(coffea-env3) [kmohrman@earth: topEFT]$ source check_yields.sh Submitting workers... Creating worker submit scripts in /tmp/kmohrman-workers... /afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/bin/condor_submit_workers: line 88: [: =: unary operator expected printf: usage: printf [-v var] format [arguments] Submitting job(s).................... 20 job(s) submitted to cluster 135978. Running processor... >> Reading json from cfg file... >> UL17_tHq_b1 - isData? : NO - year : 2017 - xsec : 0.070960 - histAxisName : tHq_privateUL17 - options : - tree : Events - nEvents : 4890500 - nGenEvents : 9818000 - SumWeights : 9818000 - Prefix : root://deepthought.crc.nd.edu/ - nFiles : 20 /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_tHq4f_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_10198.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_tHq4f_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_11205.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_tHq4f_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_12011.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_tHq4f_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_12742.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_tHq4f_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_13578.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_tHq4f_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_14255.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_tHq4f_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_14911.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_tHq4f_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_15897.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_tHq4f_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_16897.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_tHq4f_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_16910.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_tHq4f_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_16917.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_tHq4f_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_17023.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_tHq4f_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_17511.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_tHq4f_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_3146.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_tHq4f_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_4446.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_tHq4f_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_5566.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_tHq4f_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_6352.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_tHq4f_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_7196.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_tHq4f_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_8151.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_tHq4f_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_9245.root >> UL17_tllq_b1 - isData? : NO - year : 2017 - xsec : 0.075800 - histAxisName : tllq_privateUL17 - options : - tree : Events - nEvents : 4883000 - nGenEvents : 9800000 - SumWeights : 9800000 - Prefix : root://deepthought.crc.nd.edu/ - nFiles : 21 /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_tllq4fNoSchanWNoHiggs0p_all22WCsStartPtCheckV2dim6TopMay20GST_run0/NAOD-00000_10617.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_tllq4fNoSchanWNoHiggs0p_all22WCsStartPtCheckV2dim6TopMay20GST_run0/NAOD-00000_11383.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_tllq4fNoSchanWNoHiggs0p_all22WCsStartPtCheckV2dim6TopMay20GST_run0/NAOD-00000_12195.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_tllq4fNoSchanWNoHiggs0p_all22WCsStartPtCheckV2dim6TopMay20GST_run0/NAOD-00000_12943.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_tllq4fNoSchanWNoHiggs0p_all22WCsStartPtCheckV2dim6TopMay20GST_run0/NAOD-00000_13743.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_tllq4fNoSchanWNoHiggs0p_all22WCsStartPtCheckV2dim6TopMay20GST_run0/NAOD-00000_14736.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_tllq4fNoSchanWNoHiggs0p_all22WCsStartPtCheckV2dim6TopMay20GST_run0/NAOD-00000_15898.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_tllq4fNoSchanWNoHiggs0p_all22WCsStartPtCheckV2dim6TopMay20GST_run0/NAOD-00000_16904.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_tllq4fNoSchanWNoHiggs0p_all22WCsStartPtCheckV2dim6TopMay20GST_run0/NAOD-00000_17024.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_tllq4fNoSchanWNoHiggs0p_all22WCsStartPtCheckV2dim6TopMay20GST_run0/NAOD-00000_17527.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_tllq4fNoSchanWNoHiggs0p_all22WCsStartPtCheckV2dim6TopMay20GST_run0/NAOD-00000_2478.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_tllq4fNoSchanWNoHiggs0p_all22WCsStartPtCheckV2dim6TopMay20GST_run0/NAOD-00000_3623.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_tllq4fNoSchanWNoHiggs0p_all22WCsStartPtCheckV2dim6TopMay20GST_run0/NAOD-00000_4206.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_tllq4fNoSchanWNoHiggs0p_all22WCsStartPtCheckV2dim6TopMay20GST_run0/NAOD-00000_4783.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_tllq4fNoSchanWNoHiggs0p_all22WCsStartPtCheckV2dim6TopMay20GST_run0/NAOD-00000_5567.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_tllq4fNoSchanWNoHiggs0p_all22WCsStartPtCheckV2dim6TopMay20GST_run0/NAOD-00000_5897.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_tllq4fNoSchanWNoHiggs0p_all22WCsStartPtCheckV2dim6TopMay20GST_run0/NAOD-00000_6575.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_tllq4fNoSchanWNoHiggs0p_all22WCsStartPtCheckV2dim6TopMay20GST_run0/NAOD-00000_7294.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_tllq4fNoSchanWNoHiggs0p_all22WCsStartPtCheckV2dim6TopMay20GST_run0/NAOD-00000_7991.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_tllq4fNoSchanWNoHiggs0p_all22WCsStartPtCheckV2dim6TopMay20GST_run0/NAOD-00000_8727.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_tllq4fNoSchanWNoHiggs0p_all22WCsStartPtCheckV2dim6TopMay20GST_run0/NAOD-00000_9765.root >> UL17_ttHJet_b1 - isData? : NO - year : 2017 - xsec : 0.215100 - histAxisName : ttHJet_privateUL17 - options : - tree : Events - nEvents : 5280030 - nGenEvents : 26427342 - SumWeights : 26427342 - Prefix : root://deepthought.crc.nd.edu/ - nFiles : 24 /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttHJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_10199.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttHJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_11281.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttHJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_12012.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttHJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_12743.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttHJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_13390.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttHJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_14060.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttHJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_14737.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttHJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_15478.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttHJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_16125.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttHJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_16782.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttHJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_16900.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttHJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_16911.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttHJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_16914.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttHJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_16931.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttHJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_16980.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttHJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_17069.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttHJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_17534.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttHJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_3802.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttHJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_4657.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttHJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_5690.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttHJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_6435.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttHJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_7295.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttHJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_8324.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttHJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_9503.root >> UL17_ttllJet_b1 - isData? : NO - year : 2017 - xsec : 0.252900 - histAxisName : ttllJet_privateUL17 - options : - tree : Events - nEvents : 5328297 - nGenEvents : 34642133 - SumWeights : 34642133 - Prefix : root://deepthought.crc.nd.edu/ - nFiles : 22 /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttllNuNuJetNoHiggs_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_10293.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttllNuNuJetNoHiggs_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_11118.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttllNuNuJetNoHiggs_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_11917.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttllNuNuJetNoHiggs_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_12744.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttllNuNuJetNoHiggs_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_13483.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttllNuNuJetNoHiggs_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_14543.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttllNuNuJetNoHiggs_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_15782.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttllNuNuJetNoHiggs_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_16901.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttllNuNuJetNoHiggs_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_17528.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttllNuNuJetNoHiggs_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_2353.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttllNuNuJetNoHiggs_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_3624.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttllNuNuJetNoHiggs_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_3824.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttllNuNuJetNoHiggs_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_4282.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttllNuNuJetNoHiggs_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_4908.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttllNuNuJetNoHiggs_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_5335.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttllNuNuJetNoHiggs_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_5732.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttllNuNuJetNoHiggs_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_6181.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttllNuNuJetNoHiggs_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_6723.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttllNuNuJetNoHiggs_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_7390.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttllNuNuJetNoHiggs_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_8064.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttllNuNuJetNoHiggs_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_8682.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttllNuNuJetNoHiggs_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_9651.root >> UL17_ttlnuJet_b1 - isData? : NO - year : 2017 - xsec : 0.204300 - histAxisName : ttlnuJet_privateUL17 - options : - tree : Events - nEvents : 5937957 - nGenEvents : 20844397 - SumWeights : 20844397 - Prefix : root://deepthought.crc.nd.edu/ - nFiles : 26 /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttlnuJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_10294.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttlnuJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_10889.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttlnuJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_11500.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttlnuJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_12120.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttlnuJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_12639.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttlnuJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_13286.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttlnuJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_13961.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttlnuJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_14544.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttlnuJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_15379.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttlnuJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_16126.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttlnuJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_16898.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttlnuJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_16912.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttlnuJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_16955.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttlnuJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_17525.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttlnuJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_2864.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttlnuJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_3625.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttlnuJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_4106.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttlnuJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_4658.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttlnuJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_5336.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttlnuJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_5734.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttlnuJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_6353.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttlnuJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_6942.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttlnuJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_7561.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttlnuJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_8152.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttlnuJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_8793.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttlnuJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_9766.root >> UL17_tttt_b4 - isData? : NO - year : 2017 - xsec : 0.009103 - histAxisName : tttt_privateUL17 - options : - tree : Events - nEvents : 15000000 - nGenEvents : 29976500 - SumWeights : 29976500 - Prefix : root://deepthought.crc.nd.edu/ - nFiles : 82 /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_10077.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_10086.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_10087.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_10088.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_10089.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_10090.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_10091.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_10092.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_10093.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_10095.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_10096.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_10097.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_10098.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_10099.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_10100.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_10101.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_10102.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_10103.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_1161.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_1360.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_1524.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_1636.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_1756.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_1888.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_2009.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_220.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_2251.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_2372.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_2501.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_2600.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_2727.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_2858.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_2993.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_3113.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_3212.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_334.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_3348.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_3478.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_3588.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_3718.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_3858.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_3992.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_4135.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_4255.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_4405.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_4488.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_4643.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_4763.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_4880.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_5018.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_5147.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_5265.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_5405.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_5532.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_5633.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_5772.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_5912.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_6031.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_6152.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_6262.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_6383.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_6675.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_7185.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_7422.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_7784.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_7954.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_8073.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_8189.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_8329.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_8437.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_8595.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_8733.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_8816.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_8961.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_9075.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_9237.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_9291.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_9422.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_9583.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_9699.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_9848.root /store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_9962.root Wilson Coefficients: cpt, ctp, cptb, cQlMi, cQq81, cQq11, cQl3i, ctq8, ctlTi, ctq1, ctli, cQq13, cbW, cpQM, cpQ3, ctei, cQei, ctW, ctlSi, cQq83, ctZ, ctG, ctt1, cQt1, cQt8, and cQQ1. 2021-10-02 22:36:56,411:WARNING:Found unstaged changes in '/afs/crc.nd.edu/user/k/kmohrman/coffea_dir/check_PRs/master/topcoffea' 2021-10-02 22:36:56,414:INFO:Looking for base environment envs/base_env_d0db474c.tar.gz... 2021-10-02 22:36:56,415:INFO:Found in cache envs/base_env_d0db474c.tar.gz 2021-10-02 22:36:56,416:WARNING:Rebuilding environment because unstaged changes in topcoffea 2021-10-02 22:36:56,416:WARNING:Forcing rebuild of envs/full_env_d0db474c_HEAD.tar.gz 2021-10-02 22:36:56,527:INFO:Installing /afs/crc.nd.edu/user/k/kmohrman/coffea_dir/check_PRs/master/topcoffea from editable pip 2021-10-02 22:38:30,656:INFO:Generating envs/full_env_d0db474c_HEAD.tar.gz environment file Listening for work queue workers on port 9124... warning: this work queue manager is visible to the public. warning: you should set a password with the --password option. submitted preprocessing task 1 submitted preprocessing task 2 submitted preprocessing task 3 submitted preprocessing task 4 submitted preprocessing task 5 submitted preprocessing task 6 submitted preprocessing task 7 submitted preprocessing task 8 submitted preprocessing task 9 submitted preprocessing task 10 submitted preprocessing task 11 submitted preprocessing task 12 submitted preprocessing task 13 submitted preprocessing task 14 submitted preprocessing task 15 submitted preprocessing task 16 submitted preprocessing task 17 submitted preprocessing task 18 submitted preprocessing task 19 submitted preprocessing task 20 submitted preprocessing task 21 submitted preprocessing task 22 submitted preprocessing task 23 submitted preprocessing task 24 submitted preprocessing task 25 submitted preprocessing task 26 submitted preprocessing task 27 submitted preprocessing task 28 submitted preprocessing task 29 submitted preprocessing task 30 submitted preprocessing task 31 submitted preprocessing task 32 submitted preprocessing task 33 submitted preprocessing task 34 submitted preprocessing task 35 submitted preprocessing task 36 submitted preprocessing task 37 submitted preprocessing task 38 submitted preprocessing task 39 submitted preprocessing task 40 submitted preprocessing task 41 submitted preprocessing task 42 submitted preprocessing task 43 submitted preprocessing task 44 submitted preprocessing task 45 submitted preprocessing task 46 submitted preprocessing task 47 submitted preprocessing task 48 submitted preprocessing task 49 submitted preprocessing task 50 submitted preprocessing task 51 submitted preprocessing task 52 submitted preprocessing task 53 submitted preprocessing task 54 submitted preprocessing task 55 submitted preprocessing task 56 submitted preprocessing task 57 submitted preprocessing task 58 submitted preprocessing task 59 submitted preprocessing task 60 submitted preprocessing task 61 submitted preprocessing task 62 submitted preprocessing task 63 submitted preprocessing task 64 submitted preprocessing task 65 submitted preprocessing task 66 submitted preprocessing task 67 submitted preprocessing task 68 submitted preprocessing task 69 submitted preprocessing task 70 submitted preprocessing task 71 submitted preprocessing task 72 submitted preprocessing task 73 submitted preprocessing task 74 submitted preprocessing task 75 submitted preprocessing task 76 submitted preprocessing task 77 submitted preprocessing task 78 submitted preprocessing task 79 submitted preprocessing task 80 submitted preprocessing task 81 submitted preprocessing task 82 submitted preprocessing task 83 submitted preprocessing task 84 submitted preprocessing task 85 submitted preprocessing task 86 submitted preprocessing task 87 submitted preprocessing task 88 submitted preprocessing task 89 submitted preprocessing task 90 submitted preprocessing task 91 submitted preprocessing task 92 submitted preprocessing task 93 submitted preprocessing task 94 submitted preprocessing task 95 submitted preprocessing task 96 submitted preprocessing task 97 submitted preprocessing task 98 submitted preprocessing task 99 submitted preprocessing task 100 submitted preprocessing task 101 submitted preprocessing task 102 submitted preprocessing task 103 submitted preprocessing task 104 submitted preprocessing task 105 submitted preprocessing task 106 submitted preprocessing task 107 submitted preprocessing task 108 submitted preprocessing task 109 submitted preprocessing task 110 submitted preprocessing task 111 submitted preprocessing task 112 submitted preprocessing task 113 submitted preprocessing task 114 submitted preprocessing task 115 submitted preprocessing task 116 submitted preprocessing task 117 submitted preprocessing task 118 submitted preprocessing task 119 submitted preprocessing task 120 submitted preprocessing task 121 submitted preprocessing task 122 submitted preprocessing task 123 submitted preprocessing task 124 submitted preprocessing task 125 submitted preprocessing task 126 submitted preprocessing task 127 submitted preprocessing task 128 submitted preprocessing task 129 submitted preprocessing task 130 submitted preprocessing task 131 submitted preprocessing task 132 submitted preprocessing task 133 submitted preprocessing task 134 submitted preprocessing task 135 submitted preprocessing task 136 submitted preprocessing task 137 submitted preprocessing task 138 submitted preprocessing task 139 submitted preprocessing task 140 submitted preprocessing task 141 submitted preprocessing task 142 submitted preprocessing task 143 submitted preprocessing task 144 submitted preprocessing task 145 submitted preprocessing task 146 submitted preprocessing task 147 submitted preprocessing task 148 submitted preprocessing task 149 submitted preprocessing task 150 submitted preprocessing task 151 submitted preprocessing task 152 submitted preprocessing task 153 submitted preprocessing task 154 submitted preprocessing task 155 submitted preprocessing task 156 submitted preprocessing task 157 submitted preprocessing task 158 submitted preprocessing task 159 submitted preprocessing task 160 submitted preprocessing task 161 submitted preprocessing task 162 submitted preprocessing task 163 submitted preprocessing task 164 submitted preprocessing task 165 submitted preprocessing task 166 submitted preprocessing task 167 submitted preprocessing task 168 submitted preprocessing task 169 submitted preprocessing task 170 submitted preprocessing task 171 submitted preprocessing task 172 submitted preprocessing task 173 submitted preprocessing task 174 submitted preprocessing task 175 submitted preprocessing task 176 submitted preprocessing task 177 submitted preprocessing task 178 submitted preprocessing task 179 submitted preprocessing task 180 submitted preprocessing task 181 submitted preprocessing task 182 submitted preprocessing task 183 submitted preprocessing task 184 submitted preprocessing task 185 submitted preprocessing task 186 submitted preprocessing task 187 submitted preprocessing task 188 submitted preprocessing task 189 submitted preprocessing task 190 submitted preprocessing task 191 submitted preprocessing task 192 submitted preprocessing task 193 submitted preprocessing task 194 submitted preprocessing task 195 preprocessing task id 145 item pre_145 with 1 events completed on d12chas580.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.015, memory: 110.0 MB, disk 330.0 MB, gpus: 0.0, runtime 186.774574 preprocessing task id 169 item pre_169 with 1 events completed on d12chas553.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.014, memory: 108.0 MB, disk 330.0 MB, gpus: 0.0, runtime 189.020602 preprocessing task id 170 item pre_170 with 1 events completed on d12chas553.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.015, memory: 109.0 MB, disk 330.0 MB, gpus: 0.0, runtime 189.116309 preprocessing task id 148 item pre_148 with 1 events completed on d12chas587.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.015, memory: 110.0 MB, disk 330.0 MB, gpus: 0.0, runtime 188.01975 preprocessing task id 142 item pre_142 with 1 events completed on d12chas580.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.143, memory: 114.0 MB, disk 330.0 MB, gpus: 0.0, runtime 187.820589 preprocessing task id 134 item pre_134 with 1 events completed on d12chas546.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.135, memory: 110.0 MB, disk 330.0 MB, gpus: 0.0, runtime 187.029287 preprocessing task id 139 item pre_139 with 1 events completed on d12chas564.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.134, memory: 110.0 MB, disk 330.0 MB, gpus: 0.0, runtime 187.781739 preprocessing task id 124 item pre_124 with 1 events completed on d12chas564.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.015, memory: 109.0 MB, disk 330.0 MB, gpus: 0.0, runtime 186.753822 preprocessing task id 140 item pre_140 with 1 events completed on d12chas564.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.015, memory: 114.0 MB, disk 330.0 MB, gpus: 0.0, runtime 188.131703 preprocessing task id 137 item pre_137 with 1 events completed on d12chas564.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.016, memory: 111.0 MB, disk 330.0 MB, gpus: 0.0, runtime 188.18538 preprocessing task id 176 item pre_176 with 1 events completed on d12chas545.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.015, memory: 110.0 MB, disk 330.0 MB, gpus: 0.0, runtime 191.038383 preprocessing task id 136 item pre_136 with 1 events completed on d12chas564.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.015, memory: 114.0 MB, disk 330.0 MB, gpus: 0.0, runtime 188.240772 preprocessing task id 162 item pre_162 with 1 events completed on d12chas547.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.015, memory: 110.0 MB, disk 330.0 MB, gpus: 0.0, runtime 190.422144 preprocessing task id 138 item pre_138 with 1 events completed on d12chas564.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.016, memory: 113.0 MB, disk 330.0 MB, gpus: 0.0, runtime 188.397214 preprocessing task id 164 item pre_164 with 1 events completed on d12chas547.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.015, memory: 109.0 MB, disk 330.0 MB, gpus: 0.0, runtime 190.438679 preprocessing task id 165 item pre_165 with 1 events completed on d12chas547.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.014, memory: 110.0 MB, disk 330.0 MB, gpus: 0.0, runtime 190.623939 preprocessing task id 180 item pre_180 with 1 events completed on d12chas552.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.134, memory: 111.0 MB, disk 330.0 MB, gpus: 0.0, runtime 191.860932 preprocessing task id 130 item pre_130 with 1 events completed on d12chas551.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.015, memory: 109.0 MB, disk 330.0 MB, gpus: 0.0, runtime 187.804153 preprocessing task id 163 item pre_163 with 1 events completed on d12chas547.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.133, memory: 134.0 MB, disk 330.0 MB, gpus: 0.0, runtime 190.709753 preprocessing task id 128 item pre_128 with 1 events completed on d12chas551.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.135, memory: 111.0 MB, disk 330.0 MB, gpus: 0.0, runtime 187.934206 preprocessing task id 113 item pre_113 with 1 events completed on d12chas580.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.016, memory: 113.0 MB, disk 330.0 MB, gpus: 0.0, runtime 186.578825 preprocessing task id 126 item pre_126 with 1 events completed on d12chas551.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.016, memory: 110.0 MB, disk 330.0 MB, gpus: 0.0, runtime 187.949196 preprocessing task id 120 item pre_120 with 1 events completed on d12chas548.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.015, memory: 111.0 MB, disk 330.0 MB, gpus: 0.0, runtime 187.042606 preprocessing task id 117 item pre_117 with 1 events completed on d12chas548.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.015, memory: 109.0 MB, disk 330.0 MB, gpus: 0.0, runtime 187.061111 preprocessing task id 116 item pre_116 with 1 events completed on d12chas548.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.141, memory: 110.0 MB, disk 330.0 MB, gpus: 0.0, runtime 187.087841 preprocessing task id 118 item pre_118 with 1 events completed on d12chas548.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.015, memory: 113.0 MB, disk 330.0 MB, gpus: 0.0, runtime 187.114278 preprocessing task id 119 item pre_119 with 1 events completed on d12chas548.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.016, memory: 110.0 MB, disk 330.0 MB, gpus: 0.0, runtime 187.192661 preprocessing task id 110 item pre_110 with 1 events completed on d12chas553.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.015, memory: 113.0 MB, disk 330.0 MB, gpus: 0.0, runtime 186.638431 preprocessing task id 95 item pre_95 with 1 events completed on d12chas580.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.951, memory: 109.0 MB, disk 330.0 MB, gpus: 0.0, runtime 2.867939 preprocessing task id 177 item pre_177 with 1 events completed on d12chas545.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.014, memory: 100.0 MB, disk 330.0 MB, gpus: 0.0, runtime 192.113511 preprocessing task id 106 item pre_106 with 1 events completed on d12chas553.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.015, memory: 109.0 MB, disk 330.0 MB, gpus: 0.0, runtime 186.752619 preprocessing task id 93 item pre_93 with 1 events completed on d12chas553.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.954, memory: 112.0 MB, disk 330.0 MB, gpus: 0.0, runtime 2.749741 preprocessing task id 135 item pre_135 with 1 events completed on d12chas546.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.014, memory: 109.0 MB, disk 330.0 MB, gpus: 0.0, runtime 189.022408 preprocessing task id 94 item pre_94 with 1 events completed on d12chas553.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.95, memory: 111.0 MB, disk 330.0 MB, gpus: 0.0, runtime 2.867677 preprocessing task id 109 item pre_109 with 1 events completed on d12chas553.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.015, memory: 108.0 MB, disk 330.0 MB, gpus: 0.0, runtime 186.877455 preprocessing task id 7 item pre_7 with 1 events completed on qa-rtx6k-021.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.889, memory: 108.0 MB, disk 330.0 MB, gpus: 0.0, runtime 2.162107 preprocessing task id 107 item pre_107 with 1 events completed on d12chas553.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.016, memory: 110.0 MB, disk 330.0 MB, gpus: 0.0, runtime 187.315838 preprocessing task id 53 item pre_53 with 1 events completed on d12chas577.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.939, memory: 109.0 MB, disk 330.0 MB, gpus: 0.0, runtime 2.658268 preprocessing task id 52 item pre_52 with 1 events completed on d12chas577.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.931, memory: 109.0 MB, disk 330.0 MB, gpus: 0.0, runtime 2.666025 preprocessing task id 174 item pre_174 with 1 events completed on d12chas545.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.014, memory: 110.0 MB, disk 330.0 MB, gpus: 0.0, runtime 193.641273 preprocessing task id 102 item pre_102 with 1 events completed on qa-rtx6k-021.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.01, memory: 108.0 MB, disk 330.0 MB, gpus: 0.0, runtime 187.357085 preprocessing task id 2 item pre_2 with 1 events completed on qa-rtx6k-021.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.194, memory: 109.0 MB, disk 330.0 MB, gpus: 0.0, runtime 9.986686 preprocessing task id 5 item pre_5 with 1 events completed on qa-rtx6k-021.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.188, memory: 109.0 MB, disk 330.0 MB, gpus: 0.0, runtime 10.190764 preprocessing task id 4 item pre_4 with 1 events completed on qa-rtx6k-021.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.201, memory: 109.0 MB, disk 330.0 MB, gpus: 0.0, runtime 10.20198 preprocessing task id 54 item pre_54 with 1 events completed on d12chas577.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.23, memory: 109.0 MB, disk 330.0 MB, gpus: 0.0, runtime 10.352427 preprocessing task id 6 item pre_6 with 1 events completed on qa-rtx6k-021.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.195, memory: 113.0 MB, disk 330.0 MB, gpus: 0.0, runtime 10.283247 preprocessing task id 3 item pre_3 with 1 events completed on qa-rtx6k-021.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.186, memory: 108.0 MB, disk 330.0 MB, gpus: 0.0, runtime 10.458443 preprocessing task id 1 item pre_1 with 1 events completed on qa-rtx6k-021.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.186, memory: 110.0 MB, disk 330.0 MB, gpus: 0.0, runtime 10.514477 preprocessing task id 15 item pre_15 with 1 events completed on d12chas546.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.258, memory: 109.0 MB, disk 330.0 MB, gpus: 0.0, runtime 10.612179 preprocessing task id 69 item pre_69 with 1 events completed on d12chas584.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.256, memory: 108.0 MB, disk 330.0 MB, gpus: 0.0, runtime 10.700068 preprocessing task id 77 item pre_77 with 1 events completed on d12chas553.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.249, memory: 111.0 MB, disk 330.0 MB, gpus: 0.0, runtime 10.723035 preprocessing task id 21 item pre_21 with 1 events completed on d12chas546.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.272, memory: 109.0 MB, disk 330.0 MB, gpus: 0.0, runtime 10.652377 preprocessing task id 51 item pre_51 with 1 events completed on d12chas577.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.233, memory: 109.0 MB, disk 330.0 MB, gpus: 0.0, runtime 10.810504 preprocessing task id 17 item pre_17 with 1 events completed on d12chas546.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.253, memory: 111.0 MB, disk 330.0 MB, gpus: 0.0, runtime 10.759755 preprocessing task id 161 item pre_161 with 1 events completed on d12chas547.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.012, memory: 102.0 MB, disk 330.0 MB, gpus: 0.0, runtime 200.146537 preprocessing task id 50 item pre_50 with 1 events completed on d12chas577.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.244, memory: 111.0 MB, disk 330.0 MB, gpus: 0.0, runtime 10.832149 preprocessing task id 16 item pre_16 with 1 events completed on d12chas546.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.27, memory: 109.0 MB, disk 330.0 MB, gpus: 0.0, runtime 10.783992 preprocessing task id 62 item pre_62 with 1 events completed on d12chas545.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.262, memory: 113.0 MB, disk 330.0 MB, gpus: 0.0, runtime 10.869176 preprocessing task id 92 item pre_92 with 1 events completed on d12chas580.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.268, memory: 109.0 MB, disk 330.0 MB, gpus: 0.0, runtime 10.953733 preprocessing task id 12 item pre_12 with 1 events completed on d12chas589.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.268, memory: 111.0 MB, disk 330.0 MB, gpus: 0.0, runtime 10.828656 preprocessing task id 18 item pre_18 with 1 events completed on d12chas546.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.262, memory: 109.0 MB, disk 330.0 MB, gpus: 0.0, runtime 10.845165 preprocessing task id 29 item pre_29 with 1 events completed on d12chas548.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.271, memory: 110.0 MB, disk 330.0 MB, gpus: 0.0, runtime 10.870717 preprocessing task id 35 item pre_35 with 1 events completed on d12chas548.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.272, memory: 110.0 MB, disk 330.0 MB, gpus: 0.0, runtime 10.884103 preprocessing task id 73 item pre_73 with 1 events completed on d12chas553.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.271, memory: 114.0 MB, disk 330.0 MB, gpus: 0.0, runtime 10.929688 preprocessing task id 31 item pre_31 with 1 events completed on d12chas548.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.249, memory: 109.0 MB, disk 330.0 MB, gpus: 0.0, runtime 10.896175 preprocessing task id 86 item pre_86 with 1 events completed on d12chas580.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.277, memory: 109.0 MB, disk 330.0 MB, gpus: 0.0, runtime 10.99042 preprocessing task id 111 item pre_111 with 1 events completed on d12chas580.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.013, memory: 111.0 MB, disk 330.0 MB, gpus: 0.0, runtime 195.993127 preprocessing task id 182 item pre_182 with 1 events completed on d12chas584.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.014, memory: 109.0 MB, disk 330.0 MB, gpus: 0.0, runtime 201.881712 preprocessing task id 59 item pre_59 with 1 events completed on d12chas545.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.269, memory: 111.0 MB, disk 330.0 MB, gpus: 0.0, runtime 10.966124 preprocessing task id 32 item pre_32 with 1 events completed on d12chas548.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.243, memory: 109.0 MB, disk 330.0 MB, gpus: 0.0, runtime 10.931635 preprocessing task id 36 item pre_36 with 1 events completed on d12chas582.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.249, memory: 114.0 MB, disk 330.0 MB, gpus: 0.0, runtime 10.938432 preprocessing task id 45 item pre_45 with 1 events completed on d12chas552.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.254, memory: 110.0 MB, disk 330.0 MB, gpus: 0.0, runtime 10.963447 preprocessing task id 58 item pre_58 with 1 events completed on d12chas545.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.236, memory: 108.0 MB, disk 330.0 MB, gpus: 0.0, runtime 10.972799 preprocessing task id 64 item pre_64 with 1 events completed on d12chas584.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.278, memory: 110.0 MB, disk 330.0 MB, gpus: 0.0, runtime 10.974353 preprocessing task id 75 item pre_75 with 1 events completed on d12chas553.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.248, memory: 112.0 MB, disk 330.0 MB, gpus: 0.0, runtime 10.989185 preprocessing task id 57 item pre_57 with 1 events completed on d12chas545.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.253, memory: 109.0 MB, disk 330.0 MB, gpus: 0.0, runtime 11.01064 preprocessing task id 81 item pre_81 with 1 events completed on d12chas587.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.247, memory: 108.0 MB, disk 330.0 MB, gpus: 0.0, runtime 11.010041 preprocessing task id 22 item pre_22 with 1 events completed on d12chas547.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.246, memory: 108.0 MB, disk 330.0 MB, gpus: 0.0, runtime 10.97283 preprocessing task id 14 item pre_14 with 1 events completed on d12chas589.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.239, memory: 109.0 MB, disk 330.0 MB, gpus: 0.0, runtime 10.965869 preprocessing task id 83 item pre_83 with 1 events completed on d12chas587.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.253, memory: 109.0 MB, disk 330.0 MB, gpus: 0.0, runtime 11.031228 preprocessing task id 84 item pre_84 with 1 events completed on d12chas587.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.283, memory: 114.0 MB, disk 330.0 MB, gpus: 0.0, runtime 11.041817 preprocessing task id 133 item pre_133 with 1 events completed on d12chas546.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.015, memory: 114.0 MB, disk 330.0 MB, gpus: 0.0, runtime 197.82467 preprocessing task id 24 item pre_24 with 1 events completed on d12chas547.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.244, memory: 109.0 MB, disk 330.0 MB, gpus: 0.0, runtime 11.027225 preprocessing task id 55 item pre_55 with 1 events completed on d12chas577.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.232, memory: 109.0 MB, disk 330.0 MB, gpus: 0.0, runtime 11.086535 preprocessing task id 56 item pre_56 with 1 events completed on d12chas577.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.23, memory: 114.0 MB, disk 330.0 MB, gpus: 0.0, runtime 11.088611 preprocessing task id 63 item pre_63 with 1 events completed on d12chas545.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.237, memory: 108.0 MB, disk 330.0 MB, gpus: 0.0, runtime 11.112613 preprocessing task id 30 item pre_30 with 1 events completed on d12chas548.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.246, memory: 109.0 MB, disk 330.0 MB, gpus: 0.0, runtime 11.062978 preprocessing task id 25 item pre_25 with 1 events completed on d12chas547.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.257, memory: 109.0 MB, disk 330.0 MB, gpus: 0.0, runtime 11.064633 preprocessing task id 68 item pre_68 with 1 events completed on d12chas584.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.268, memory: 109.0 MB, disk 330.0 MB, gpus: 0.0, runtime 11.10801 preprocessing task id 41 item pre_41 with 1 events completed on d12chas582.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.254, memory: 114.0 MB, disk 330.0 MB, gpus: 0.0, runtime 11.114506 preprocessing task id 90 item pre_90 with 1 events completed on d12chas580.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.263, memory: 114.0 MB, disk 330.0 MB, gpus: 0.0, runtime 11.224766 preprocessing task id 141 item pre_141 with 1 events completed on d12chas580.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.015, memory: 114.0 MB, disk 330.0 MB, gpus: 0.0, runtime 198.858069 preprocessing task id 72 item pre_72 with 1 events completed on d12chas553.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.244, memory: 114.0 MB, disk 330.0 MB, gpus: 0.0, runtime 11.232533 preprocessing task id 80 item pre_80 with 1 events completed on d12chas587.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.26, memory: 109.0 MB, disk 330.0 MB, gpus: 0.0, runtime 11.260369 preprocessing task id 144 item pre_144 with 1 events completed on d12chas580.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.015, memory: 111.0 MB, disk 330.0 MB, gpus: 0.0, runtime 198.920568 preprocessing task id 65 item pre_65 with 1 events completed on d12chas584.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.253, memory: 114.0 MB, disk 330.0 MB, gpus: 0.0, runtime 11.258491 preprocessing task id 79 item pre_79 with 1 events completed on d12chas587.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.257, memory: 114.0 MB, disk 330.0 MB, gpus: 0.0, runtime 11.28754 preprocessing task id 9 item pre_9 with 1 events completed on d12chas589.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.253, memory: 109.0 MB, disk 330.0 MB, gpus: 0.0, runtime 11.177933 preprocessing task id 112 item pre_112 with 1 events completed on d12chas580.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.015, memory: 113.0 MB, disk 330.0 MB, gpus: 0.0, runtime 196.336272 preprocessing task id 42 item pre_42 with 1 events completed on d12chas582.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.262, memory: 110.0 MB, disk 330.0 MB, gpus: 0.0, runtime 11.268154 preprocessing task id 39 item pre_39 with 1 events completed on d12chas582.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.238, memory: 109.0 MB, disk 330.0 MB, gpus: 0.0, runtime 11.290039 preprocessing task id 27 item pre_27 with 1 events completed on d12chas547.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.231, memory: 108.0 MB, disk 330.0 MB, gpus: 0.0, runtime 11.271693 preprocessing task id 34 item pre_34 with 1 events completed on d12chas548.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.248, memory: 114.0 MB, disk 330.0 MB, gpus: 0.0, runtime 11.283959 preprocessing task id 28 item pre_28 with 1 events completed on d12chas547.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.275, memory: 110.0 MB, disk 330.0 MB, gpus: 0.0, runtime 11.289292 preprocessing task id 40 item pre_40 with 1 events completed on d12chas582.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.294, memory: 109.0 MB, disk 330.0 MB, gpus: 0.0, runtime 11.310532 preprocessing task id 43 item pre_43 with 1 events completed on d12chas552.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.238, memory: 109.0 MB, disk 330.0 MB, gpus: 0.0, runtime 11.317011 preprocessing task id 70 item pre_70 with 1 events completed on d12chas584.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.236, memory: 109.0 MB, disk 330.0 MB, gpus: 0.0, runtime 11.377079 preprocessing task id 47 item pre_47 with 1 events completed on d12chas552.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.267, memory: 114.0 MB, disk 330.0 MB, gpus: 0.0, runtime 11.351304 preprocessing task id 76 item pre_76 with 1 events completed on d12chas553.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.257, memory: 111.0 MB, disk 330.0 MB, gpus: 0.0, runtime 11.363457 preprocessing task id 67 item pre_67 with 1 events completed on d12chas584.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.255, memory: 109.0 MB, disk 330.0 MB, gpus: 0.0, runtime 11.391893 preprocessing task id 74 item pre_74 with 1 events completed on d12chas553.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.266, memory: 109.0 MB, disk 330.0 MB, gpus: 0.0, runtime 11.407297 preprocessing task id 87 item pre_87 with 1 events completed on d12chas580.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.234, memory: 109.0 MB, disk 330.0 MB, gpus: 0.0, runtime 11.425503 preprocessing task id 26 item pre_26 with 1 events completed on d12chas547.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.265, memory: 114.0 MB, disk 330.0 MB, gpus: 0.0, runtime 11.347194 preprocessing task id 105 item pre_105 with 1 events completed on qa-rtx6k-021.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.09, memory: 110.0 MB, disk 330.0 MB, gpus: 0.0, runtime 194.850241 preprocessing task id 188 item pre_188 with 1 events completed on d12chas577.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.013, memory: 110.0 MB, disk 330.0 MB, gpus: 0.0, runtime 205.827323 preprocessing task id 23 item pre_23 with 1 events completed on d12chas547.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.242, memory: 113.0 MB, disk 330.0 MB, gpus: 0.0, runtime 11.373736 preprocessing task id 143 item pre_143 with 1 events completed on d12chas580.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.014, memory: 106.0 MB, disk 330.0 MB, gpus: 0.0, runtime 199.117211 preprocessing task id 89 item pre_89 with 1 events completed on d12chas580.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.239, memory: 108.0 MB, disk 330.0 MB, gpus: 0.0, runtime 11.507849 preprocessing task id 10 item pre_10 with 1 events completed on d12chas589.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.245, memory: 109.0 MB, disk 330.0 MB, gpus: 0.0, runtime 11.384665 preprocessing task id 61 item pre_61 with 1 events completed on d12chas545.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.24, memory: 109.0 MB, disk 330.0 MB, gpus: 0.0, runtime 11.470031 preprocessing task id 49 item pre_49 with 1 events completed on d12chas552.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.241, memory: 112.0 MB, disk 330.0 MB, gpus: 0.0, runtime 11.46645 preprocessing task id 78 item pre_78 with 1 events completed on d12chas587.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.242, memory: 109.0 MB, disk 330.0 MB, gpus: 0.0, runtime 11.500152 preprocessing task id 160 item pre_160 with 1 events completed on d12chas546.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.014, memory: 108.0 MB, disk 330.0 MB, gpus: 0.0, runtime 200.385092 preprocessing task id 11 item pre_11 with 1 events completed on d12chas589.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.246, memory: 109.0 MB, disk 330.0 MB, gpus: 0.0, runtime 11.423355 preprocessing task id 48 item pre_48 with 1 events completed on d12chas552.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.247, memory: 110.0 MB, disk 330.0 MB, gpus: 0.0, runtime 11.479127 preprocessing task id 114 item pre_114 with 1 events completed on d12chas580.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.015, memory: 111.0 MB, disk 330.0 MB, gpus: 0.0, runtime 196.584545 preprocessing task id 82 item pre_82 with 1 events completed on d12chas587.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.241, memory: 114.0 MB, disk 330.0 MB, gpus: 0.0, runtime 11.568717 preprocessing task id 8 item pre_8 with 1 events completed on d12chas589.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.253, memory: 109.0 MB, disk 330.0 MB, gpus: 0.0, runtime 11.462188 preprocessing task id 66 item pre_66 with 1 events completed on d12chas584.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.241, memory: 108.0 MB, disk 330.0 MB, gpus: 0.0, runtime 11.564108 preprocessing task id 33 item pre_33 with 1 events completed on d12chas548.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.243, memory: 113.0 MB, disk 330.0 MB, gpus: 0.0, runtime 11.541492 preprocessing task id 20 item pre_20 with 1 events completed on d12chas546.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.245, memory: 109.0 MB, disk 330.0 MB, gpus: 0.0, runtime 11.540998 preprocessing task id 13 item pre_13 with 1 events completed on d12chas589.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.309, memory: 111.0 MB, disk 330.0 MB, gpus: 0.0, runtime 11.531834 preprocessing task id 19 item pre_19 with 1 events completed on d12chas546.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.244, memory: 110.0 MB, disk 330.0 MB, gpus: 0.0, runtime 11.564492 preprocessing task id 37 item pre_37 with 1 events completed on d12chas582.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.244, memory: 109.0 MB, disk 330.0 MB, gpus: 0.0, runtime 11.597234 preprocessing task id 44 item pre_44 with 1 events completed on d12chas552.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.269, memory: 108.0 MB, disk 330.0 MB, gpus: 0.0, runtime 11.6172 preprocessing task id 71 item pre_71 with 1 events completed on d12chas553.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.241, memory: 111.0 MB, disk 330.0 MB, gpus: 0.0, runtime 11.660184 preprocessing task id 46 item pre_46 with 1 events completed on d12chas552.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.289, memory: 108.0 MB, disk 330.0 MB, gpus: 0.0, runtime 11.669253 preprocessing task id 60 item pre_60 with 1 events completed on d12chas545.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.29, memory: 113.0 MB, disk 330.0 MB, gpus: 0.0, runtime 11.709656 preprocessing task id 91 item pre_91 with 1 events completed on d12chas580.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.242, memory: 110.0 MB, disk 330.0 MB, gpus: 0.0, runtime 11.764845 preprocessing task id 38 item pre_38 with 1 events completed on d12chas582.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.244, memory: 110.0 MB, disk 330.0 MB, gpus: 0.0, runtime 11.689379 preprocessing task id 85 item pre_85 with 1 events completed on d12chas587.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.248, memory: 111.0 MB, disk 330.0 MB, gpus: 0.0, runtime 11.770565 preprocessing task id 104 item pre_104 with 1 events completed on qa-rtx6k-021.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.01, memory: 110.0 MB, disk 330.0 MB, gpus: 0.0, runtime 195.225864 preprocessing task id 157 item pre_157 with 1 events completed on d12chas546.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.015, memory: 114.0 MB, disk 330.0 MB, gpus: 0.0, runtime 200.716453 preprocessing task id 88 item pre_88 with 1 events completed on d12chas580.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.243, memory: 114.0 MB, disk 330.0 MB, gpus: 0.0, runtime 11.91925 preprocessing task id 190 item pre_190 with 1 events completed on d12chas577.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.013, memory: 109.0 MB, disk 330.0 MB, gpus: 0.0, runtime 207.584874 preprocessing task id 173 item pre_173 with 1 events completed on d12chas552.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.014, memory: 109.0 MB, disk 330.0 MB, gpus: 0.0, runtime 203.575801 preprocessing task id 123 item pre_123 with 1 events completed on d12chas564.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.125, memory: 110.0 MB, disk 330.0 MB, gpus: 0.0, runtime 200.395753 preprocessing task id 151 item pre_151 with 1 events completed on d12chas589.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.122, memory: 110.0 MB, disk 330.0 MB, gpus: 0.0, runtime 203.995218 preprocessing task id 154 item pre_154 with 1 events completed on d12chas589.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.015, memory: 110.0 MB, disk 330.0 MB, gpus: 0.0, runtime 204.72583 preprocessing task id 99 item pre_99 with 1 events completed on qa-rtx6k-021.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.09, memory: 110.0 MB, disk 330.0 MB, gpus: 0.0, runtime 199.359719 preprocessing task id 98 item pre_98 with 1 events completed on qa-rtx6k-021.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.01, memory: 111.0 MB, disk 330.0 MB, gpus: 0.0, runtime 199.430311 preprocessing task id 100 item pre_100 with 1 events completed on qa-rtx6k-021.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.01, memory: 108.0 MB, disk 330.0 MB, gpus: 0.0, runtime 199.447413 preprocessing task id 159 item pre_159 with 1 events completed on d12chas546.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.013, memory: 108.0 MB, disk 330.0 MB, gpus: 0.0, runtime 205.891566 preprocessing task id 181 item pre_181 with 1 events completed on d12chas584.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.123, memory: 110.0 MB, disk 330.0 MB, gpus: 0.0, runtime 207.951318 preprocessing task id 108 item pre_108 with 1 events completed on d12chas553.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.121, memory: 110.0 MB, disk 330.0 MB, gpus: 0.0, runtime 201.69147 preprocessing task id 185 item pre_185 with 1 events completed on d12chas584.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.013, memory: 114.0 MB, disk 330.0 MB, gpus: 0.0, runtime 209.527508 preprocessing task id 131 item pre_131 with 1 events completed on d12chas546.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.013, memory: 109.0 MB, disk 330.0 MB, gpus: 0.0, runtime 207.080625 preprocessing task id 179 item pre_179 with 1 events completed on d12chas552.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.013, memory: 109.0 MB, disk 330.0 MB, gpus: 0.0, runtime 210.978113 preprocessing task id 186 item pre_186 with 1 events completed on d12chas577.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.012, memory: 110.0 MB, disk 330.0 MB, gpus: 0.0, runtime 214.89465 preprocessing task id 189 item pre_189 with 1 events completed on d12chas577.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.105, memory: 114.0 MB, disk 330.0 MB, gpus: 0.0, runtime 214.921071 preprocessing task id 187 item pre_187 with 1 events completed on d12chas577.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.013, memory: 109.0 MB, disk 330.0 MB, gpus: 0.0, runtime 214.925989 preprocessing task id 175 item pre_175 with 1 events completed on d12chas545.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.121, memory: 134.0 MB, disk 330.0 MB, gpus: 0.0, runtime 210.792915 preprocessing task id 172 item pre_172 with 1 events completed on d12chas552.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.014, memory: 111.0 MB, disk 330.0 MB, gpus: 0.0, runtime 210.887909 preprocessing task id 103 item pre_103 with 1 events completed on qa-rtx6k-021.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.009, memory: 109.0 MB, disk 330.0 MB, gpus: 0.0, runtime 206.340299 preprocessing task id 129 item pre_129 with 1 events completed on d12chas551.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.013, memory: 109.0 MB, disk 330.0 MB, gpus: 0.0, runtime 209.333921 preprocessing task id 115 item pre_115 with 1 events completed on d12chas580.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.126, memory: 110.0 MB, disk 330.0 MB, gpus: 0.0, runtime 208.051347 preprocessing task id 155 item pre_155 with 1 events completed on d12chas589.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.013, memory: 109.0 MB, disk 330.0 MB, gpus: 0.0, runtime 211.479701 preprocessing task id 184 item pre_184 with 1 events completed on d12chas584.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.012, memory: 103.0 MB, disk 330.0 MB, gpus: 0.0, runtime 214.078833 preprocessing task id 153 item pre_153 with 1 events completed on d12chas589.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.013, memory: 114.0 MB, disk 330.0 MB, gpus: 0.0, runtime 211.795935 preprocessing task id 152 item pre_152 with 1 events completed on d12chas589.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.014, memory: 111.0 MB, disk 330.0 MB, gpus: 0.0, runtime 211.799989 preprocessing task id 97 item pre_97 with 1 events completed on qa-rtx6k-021.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.009, memory: 111.0 MB, disk 330.0 MB, gpus: 0.0, runtime 206.296737 preprocessing task id 101 item pre_101 with 1 events completed on qa-rtx6k-021.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.009, memory: 109.0 MB, disk 330.0 MB, gpus: 0.0, runtime 206.829547 preprocessing task id 168 item pre_168 with 1 events completed on d12chas553.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.012, memory: 109.0 MB, disk 330.0 MB, gpus: 0.0, runtime 213.279582 preprocessing task id 183 item pre_183 with 1 events completed on d12chas584.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.013, memory: 110.0 MB, disk 330.0 MB, gpus: 0.0, runtime 214.626844 preprocessing task id 96 item pre_96 with 1 events completed on qa-rtx6k-021.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.009, memory: 114.0 MB, disk 330.0 MB, gpus: 0.0, runtime 206.800297 preprocessing task id 166 item pre_166 with 1 events completed on d12chas552.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.013, memory: 109.0 MB, disk 330.0 MB, gpus: 0.0, runtime 214.05478 preprocessing task id 150 item pre_150 with 1 events completed on d12chas587.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.013, memory: 110.0 MB, disk 330.0 MB, gpus: 0.0, runtime 213.155098 preprocessing task id 194 item pre_194 with 1 events completed on d12chas582.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.012, memory: 111.0 MB, disk 330.0 MB, gpus: 0.0, runtime 220.290548 preprocessing task id 147 item pre_147 with 1 events completed on d12chas587.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.121, memory: 134.0 MB, disk 330.0 MB, gpus: 0.0, runtime 213.309884 preprocessing task id 121 item pre_121 with 1 events completed on d12chas564.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.012, memory: 109.0 MB, disk 330.0 MB, gpus: 0.0, runtime 211.402028 preprocessing task id 149 item pre_149 with 1 events completed on d12chas587.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.013, memory: 113.0 MB, disk 330.0 MB, gpus: 0.0, runtime 213.434267 preprocessing task id 193 item pre_193 with 1 events completed on d12chas582.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.111, memory: 134.0 MB, disk 330.0 MB, gpus: 0.0, runtime 220.633986 preprocessing task id 122 item pre_122 with 1 events completed on d12chas564.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.013, memory: 110.0 MB, disk 330.0 MB, gpus: 0.0, runtime 211.794744 preprocessing task id 191 item pre_191 with 1 events completed on d12chas582.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.012, memory: 109.0 MB, disk 330.0 MB, gpus: 0.0, runtime 220.899934 preprocessing task id 195 item pre_195 with 1 events completed on d12chas582.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.013, memory: 114.0 MB, disk 330.0 MB, gpus: 0.0, runtime 221.057436 preprocessing task id 146 item pre_146 with 1 events completed on d12chas587.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.015, memory: 113.0 MB, disk 330.0 MB, gpus: 0.0, runtime 214.285051 preprocessing task id 192 item pre_192 with 1 events completed on d12chas582.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.013, memory: 112.0 MB, disk 330.0 MB, gpus: 0.0, runtime 221.500057 preprocessing task id 125 item pre_125 with 1 events completed on d12chas564.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.013, memory: 112.0 MB, disk 330.0 MB, gpus: 0.0, runtime 212.649725 preprocessing task id 158 item pre_158 with 1 events completed on d12chas546.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.013, memory: 109.0 MB, disk 330.0 MB, gpus: 0.0, runtime 216.417018 preprocessing task id 156 item pre_156 with 1 events completed on d12chas546.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.121, memory: 111.0 MB, disk 330.0 MB, gpus: 0.0, runtime 217.460582 preprocessing task id 127 item pre_127 with 1 events completed on d12chas551.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.013, memory: 114.0 MB, disk 330.0 MB, gpus: 0.0, runtime 215.718678 preprocessing task id 132 item pre_132 with 1 events completed on d12chas546.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.014, memory: 114.0 MB, disk 330.0 MB, gpus: 0.0, runtime 216.340387 preprocessing task id 167 item pre_167 with 1 events completed on d12chas553.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.126, memory: 111.0 MB, disk 330.0 MB, gpus: 0.0, runtime 219.773511 preprocessing task id 171 item pre_171 with 1 events completed on d12chas553.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.013, memory: 113.0 MB, disk 330.0 MB, gpus: 0.0, runtime 219.777787 preprocessing task id 178 item pre_178 with 1 events completed on d12chas545.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.011, memory: 109.0 MB, disk 330.0 MB, gpus: 0.0, runtime 264.870499 Preprocessing 100%|███████████████████████████████████████████████████████| 195/195 [05:06<00:00, 1.57s/file] submitted processing task id 196 item 38e3a825, with 94500 events submitted processing task id 197 item 23e96f34, with 94500 events submitted processing task id 198 item 6ee65d27, with 94500 events submitted processing task id 199 item 4a3a7388, with 94500 events submitted processing task id 200 item a4e6a56b, with 94500 events submitted processing task id 201 item 1aef3caa, with 94500 events submitted processing task id 202 item 6c4c96dd, with 94500 events submitted processing task id 203 item ef13c2ec, with 94500 events submitted processing task id 204 item c8375867, with 90500 events submitted processing task id 205 item fda16d44, with 90500 events submitted processing task id 206 item 4fb6e502, with 88750 events submitted processing task id 207 item d2a35077, with 88750 events submitted processing task id 208 item 13567a1f, with 94250 events submitted processing task id 209 item 6e13def0, with 94250 events submitted processing task id 210 item 3477d0b0, with 89750 events submitted processing task id 211 item c9eba7ea, with 89750 events submitted processing task id 212 item 49907e89, with 94500 events submitted processing task id 213 item ff33ebf0, with 94500 events submitted processing task id 214 item d873aa4f, with 94500 events submitted processing task id 215 item 9a2265fb, with 94500 events submitted processing task id 216 item f6b2e414, with 94250 events submitted processing task id 217 item e3d1d1ef, with 94250 events submitted processing task id 218 item 5a3ec94f, with 94500 events submitted processing task id 219 item ae1f95d8, with 94500 events submitted processing task id 220 item b2f50fdb, with 94500 events submitted processing task id 221 item 883fe191, with 94500 events submitted processing task id 222 item f558ed3b, with 94500 events submitted processing task id 223 item 0ff91ae0, with 94500 events submitted processing task id 224 item 93b9bb90, with 94500 events submitted processing task id 225 item 8e204fb6, with 94500 events submitted processing task id 226 item f2e0c885, with 94500 events submitted processing task id 227 item ee247cbb, with 94500 events submitted processing task id 228 item 2cfd5e91, with 94500 events submitted processing task id 229 item ebdcddd9, with 94500 events submitted processing task id 230 item a77fbeae, with 94500 events submitted processing task id 231 item 9bbfe237, with 94500 events submitted processing task id 232 item a479ea26, with 94500 events submitted processing task id 233 item 6f75b793, with 94500 events submitted processing task id 234 item 2bab3509, with 91750 events submitted processing task id 235 item 7ec52b05, with 91750 events submitted processing task id 236 item 60c34c7d, with 94500 events submitted processing task id 237 item e654b357, with 94500 events submitted processing task id 238 item 60935b71, with 89000 events submitted processing task id 239 item f6b69664, with 89000 events submitted processing task id 240 item e8798c80, with 94500 events submitted processing task id 241 item d97ed809, with 94500 events submitted processing task id 242 item 69c27f6c, with 93500 events submitted processing task id 243 item 51688f84, with 93500 events submitted processing task id 244 item 9b51a0e2, with 94500 events submitted processing task id 245 item a6d3e0ea, with 94500 events submitted processing task id 246 item 2912ed25, with 90000 events submitted processing task id 247 item 8fdbebf2, with 90000 events submitted processing task id 248 item e670ff3d, with 94500 events submitted processing task id 249 item a5a933ab, with 94500 events submitted processing task id 250 item 0f43b78f, with 93000 events submitted processing task id 251 item 4f7bffd9, with 93000 events submitted processing task id 252 item 7c67d602, with 94500 events submitted processing task id 253 item 22401c7d, with 94500 events submitted processing task id 254 item fba5b9b2, with 94500 events submitted processing task id 255 item cdf636af, with 94500 events submitted processing task id 256 item 551ae671, with 94500 events submitted processing task id 257 item e6dc8ed0, with 94500 events submitted processing task id 258 item fba22e1c, with 91250 events submitted processing task id 259 item 10b7f425, with 91250 events submitted processing task id 260 item fdea3f31, with 94500 events submitted processing task id 261 item 17774384, with 94500 events submitted processing task id 262 item 3ed6bd11, with 90500 events submitted processing task id 263 item aebf33fd, with 90500 events submitted processing task id 264 item 659f2f5c, with 91250 events submitted processing task id 265 item 98152ef3, with 91250 events submitted processing task id 266 item 56aa81bf, with 87000 events submitted processing task id 267 item 780bd2de, with 87000 events submitted processing task id 268 item 7dbbcea3, with 86500 events submitted processing task id 269 item 6ba551a9, with 86500 events submitted processing task id 270 item 520f8ac6, with 94500 events submitted processing task id 271 item 85ed4354, with 94500 events submitted processing task id 272 item 1e1a7235, with 94500 events submitted processing task id 273 item 61f474b0, with 94500 events submitted processing task id 274 item e90d97f0, with 94500 events submitted processing task id 275 item d59550a1, with 94500 events submitted processing task id 276 item d0b28d4b, with 94500 events submitted processing task id 277 item ba20c181, with 94500 events submitted processing task id 278 item 8a437492, with 94500 events submitted processing task id 279 item 4ac96a4a, with 94500 events submitted processing task id 280 item df2ee331, with 94500 events submitted processing task id 281 item 2404266e, with 94500 events submitted processing task id 282 item 75254b3e, with 86750 events submitted processing task id 283 item 61dd0dc1, with 86750 events submitted processing task id 284 item b1fde89d, with 94500 events submitted processing task id 285 item a00c4e65, with 94500 events submitted processing task id 286 item 6a687e18, with 94500 events submitted processing task id 287 item 98bcdcde, with 94500 events submitted processing task id 288 item 7ee8cf1f, with 85500 events submitted processing task id 289 item a4c954ce, with 85500 events submitted processing task id 290 item 85b4efc6, with 87500 events submitted processing task id 291 item 6a1c4b1f, with 87500 events submitted processing task id 292 item e6accc12, with 94500 events submitted processing task id 293 item e0abc297, with 94500 events submitted processing task id 294 item 4b082725, with 94500 events submitted processing task id 295 item 5b25d527, with 94500 events submitted processing task id 296 item 10d36d02, with 94500 events submitted processing task id 297 item e442d6ab, with 94500 events submitted processing task id 298 item a43a86e9, with 94500 events submitted processing task id 299 item ebfa1e03, with 94500 events submitted processing task id 300 item 0deaad7a, with 85250 events submitted processing task id 301 item dcd605ae, with 85250 events submitted processing task id 302 item b0cf619e, with 85250 events submitted processing task id 303 item e0f9c340, with 85250 events submitted processing task id 304 item 97aa439d, with 92750 events submitted processing task id 305 item 7dd0d1a7, with 92750 events processing task id 205 item fda16d44 with 90500 events completed on d12chas553.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.743, memory: 1881.0 MB, disk 354.0 MB, gpus: 0.0, runtime 92.542382 submitted processing task id 306 item 159cc5c9, with 87000 events processing task id 217 item e3d1d1ef with 94250 events completed on qa-rtx6k-021.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.672, memory: 1166.0 MB, disk 354.0 MB, gpus: 0.0, runtime 85.77028 processing task id 219 item ae1f95d8 with 94500 events completed on qa-rtx6k-021.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.621, memory: 1167.0 MB, disk 354.0 MB, gpus: 0.0, runtime 88.376787 submitted processing task id 307 item 48271323, with 87000 events processing task id 216 item f6b2e414 with 94250 events completed on qa-rtx6k-021.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.629, memory: 1107.0 MB, disk 354.0 MB, gpus: 0.0, runtime 89.366364 submitted processing task id 308 item ed569bb6, with 94500 events processing task id 204 item c8375867 with 90500 events completed on d12chas553.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.645, memory: 1077.0 MB, disk 354.0 MB, gpus: 0.0, runtime 103.59081 submitted processing task id 309 item 45204ecc, with 94500 events submitted processing task id 310 item c13c0a36, with 94500 events submitted processing task id 311 item 6160f7b1, with 94500 events submitted processing task id 312 item 97c4bb33, with 94500 events submitted processing task id 313 item d11cd9ba, with 94500 events submitted processing task id 314 item 47daae98, with 85500 events submitted processing task id 315 item 56830adb, with 85500 events submitted processing task id 316 item 07e73a69, with 87500 events submitted processing task id 317 item e3e82e83, with 87500 events submitted processing task id 318 item c3ca71c8, with 94500 events processing task id 207 item d2a35077 with 88750 events completed on d12chas547.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.689, memory: 1445.0 MB, disk 354.0 MB, gpus: 0.0, runtime 97.876239 processing task id 211 item c9eba7ea with 89750 events completed on d12chas548.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.703, memory: 1117.0 MB, disk 354.0 MB, gpus: 0.0, runtime 98.322831 submitted processing task id 319 item 859a646b, with 94500 events submitted processing task id 320 item c7abad5d, with 94500 events submitted processing task id 321 item ca251bc6, with 94500 events submitted processing task id 322 item 58262c43, with 86750 events submitted processing task id 323 item 1aff935a, with 86750 events submitted processing task id 324 item f7696514, with 27000 events submitted processing task id 325 item 950c1f9b, with 85500 events submitted processing task id 326 item a2d35269, with 85500 events submitted processing task id 327 item 2b508358, with 86000 events submitted processing task id 328 item df24000d, with 86000 events processing task id 202 item 6c4c96dd with 94500 events completed on d12chas553.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.693, memory: 1161.0 MB, disk 354.0 MB, gpus: 0.0, runtime 104.803757 submitted processing task id 329 item 8a469fac, with 87000 events submitted processing task id 330 item df8ada55, with 87000 events submitted processing task id 331 item 33e0e901, with 93750 events submitted processing task id 332 item 31c18953, with 93750 events submitted processing task id 333 item 23498b27, with 89750 events submitted processing task id 334 item e9b53f28, with 89750 events submitted processing task id 335 item 154b7686, with 94500 events submitted processing task id 336 item eec901e9, with 94500 events submitted processing task id 337 item cdf22113, with 89000 events submitted processing task id 338 item 10c1bdfc, with 89000 events processing task id 200 item a4e6a56b with 94500 events completed on d12chas580.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.69, memory: 1166.0 MB, disk 354.0 MB, gpus: 0.0, runtime 107.409957 submitted processing task id 339 item 0b1bcd41, with 94250 events submitted processing task id 340 item b404b6c2, with 94250 events submitted processing task id 341 item 7d52b268, with 94500 events submitted processing task id 342 item 29c9b6ca, with 94500 events submitted processing task id 343 item 3e4ad448, with 94500 events submitted processing task id 344 item d4f5ccfc, with 94500 events submitted processing task id 345 item bfa76817, with 94500 events submitted processing task id 346 item 27aaaee0, with 94500 events submitted processing task id 347 item 3e008e90, with 94250 events submitted processing task id 348 item 5fcba9fa, with 94250 events processing task id 209 item 6e13def0 with 94250 events completed on d12chas547.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.718, memory: 2045.0 MB, disk 354.0 MB, gpus: 0.0, runtime 103.138163 submitted processing task id 349 item 7929f3f3, with 94500 events submitted processing task id 350 item 43010f60, with 94500 events submitted processing task id 351 item 2c231605, with 94500 events submitted processing task id 352 item 34a284aa, with 94500 events submitted processing task id 353 item 056212f9, with 94500 events submitted processing task id 354 item e5241d59, with 94500 events submitted processing task id 355 item 0e2e8207, with 94500 events submitted processing task id 356 item 01a58bfd, with 94500 events processing task id 197 item 23e96f34 with 94500 events completed on d12chas580.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.715, memory: 1111.0 MB, disk 355.0 MB, gpus: 0.0, runtime 109.258996 processing task id 196 item 38e3a825 with 94500 events completed on d12chas580.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.683, memory: 1168.0 MB, disk 355.0 MB, gpus: 0.0, runtime 109.855748 processing task id 206 item 4fb6e502 with 88750 events completed on d12chas547.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.632, memory: 1058.0 MB, disk 354.0 MB, gpus: 0.0, runtime 104.592676 submitted processing task id 357 item 8e4ec4f4, with 94500 events submitted processing task id 358 item 6cf8a98a, with 94500 events submitted processing task id 359 item 044f2f1f, with 119113 events submitted processing task id 360 item d55cb577, with 119113 events submitted processing task id 361 item 99be0d67, with 113392 events submitted processing task id 362 item 46e81f1b, with 113392 events submitted processing task id 363 item ba70ab41, with 113080 events processing task id 225 item 8e204fb6 with 94500 events completed on d12chas587.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.71, memory: 1166.0 MB, disk 355.0 MB, gpus: 0.0, runtime 100.813122 submitted processing task id 364 item 2da82cb2, with 113079 events processing task id 231 item 9bbfe237 with 94500 events completed on d12chas552.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.744, memory: 1088.0 MB, disk 355.0 MB, gpus: 0.0, runtime 96.447803 submitted processing task id 365 item 5bee00c9, with 118381 events submitted processing task id 366 item b9125432, with 118381 events submitted processing task id 367 item ad705893, with 115080 events submitted processing task id 368 item 7434bd8e, with 115079 events submitted processing task id 369 item 397c23ee, with 112549 events processing task id 235 item 7ec52b05 with 91750 events completed on d12chas552.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.752, memory: 2115.0 MB, disk 354.0 MB, gpus: 0.0, runtime 96.77039 processing task id 230 item a77fbeae with 94500 events completed on d12chas577.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.743, memory: 2251.0 MB, disk 355.0 MB, gpus: 0.0, runtime 96.82199 processing task id 213 item ff33ebf0 with 94500 events completed on d12chas548.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.656, memory: 1517.0 MB, disk 354.0 MB, gpus: 0.0, runtime 108.061797 submitted processing task id 370 item b471bce2, with 112549 events submitted processing task id 371 item ba606236, with 119522 events submitted processing task id 372 item c712bfc5, with 119522 events submitted processing task id 373 item 42b3e2b8, with 119233 events submitted processing task id 374 item d84b939c, with 119232 events submitted processing task id 375 item fa3eb9f8, with 119157 events submitted processing task id 376 item c2ca64a7, with 119156 events submitted processing task id 377 item 57830eba, with 119218 events submitted processing task id 378 item fae745f2, with 119217 events submitted processing task id 379 item e9636046, with 119248 events processing task id 222 item f558ed3b with 94500 events completed on d12chas587.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.684, memory: 1685.0 MB, disk 355.0 MB, gpus: 0.0, runtime 103.422941 submitted processing task id 380 item 9d220c0e, with 119248 events submitted processing task id 381 item 312973ef, with 119598 events submitted processing task id 382 item e2a5c979, with 119598 events submitted processing task id 383 item 61a912bc, with 138451 events submitted processing task id 384 item 983e90d7, with 108926 events processing task id 227 item ee247cbb with 94500 events completed on d12chas577.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.656, memory: 1111.0 MB, disk 355.0 MB, gpus: 0.0, runtime 99.444898 submitted processing task id 385 item 14c0da6c, with 108926 events submitted processing task id 386 item 8d469b3d, with 119285 events submitted processing task id 387 item c9b9f8d0, with 119285 events processing task id 223 item 0ff91ae0 with 94500 events completed on d12chas587.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.68, memory: 1110.0 MB, disk 355.0 MB, gpus: 0.0, runtime 105.095778 submitted processing task id 388 item fdbf65b4, with 116057 events submitted processing task id 389 item 5b67d2af, with 116056 events submitted processing task id 390 item 566bf5de, with 112239 events submitted processing task id 391 item fd8b6091, with 112238 events processing task id 234 item 2bab3509 with 91750 events completed on d12chas552.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.66, memory: 2116.0 MB, disk 354.0 MB, gpus: 0.0, runtime 102.289392 submitted processing task id 392 item d8131585, with 117530 events processing task id 212 item 49907e89 with 94500 events completed on d12chas548.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.674, memory: 1110.0 MB, disk 355.0 MB, gpus: 0.0, runtime 115.328727 submitted processing task id 393 item 97ee91e3, with 117530 events processing task id 242 item 69c27f6c with 93500 events completed on d12chas546.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.768, memory: 1458.0 MB, disk 354.0 MB, gpus: 0.0, runtime 101.132342 submitted processing task id 394 item 76a6d86f, with 117615 events processing task id 236 item 60c34c7d with 94500 events completed on d12chas589.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.696, memory: 1093.0 MB, disk 355.0 MB, gpus: 0.0, runtime 101.338876 submitted accumulation task id 395 item accum_1, with 2331000 events processing task id 240 item e8798c80 with 94500 events completed on d12chas589.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.678, memory: 1108.0 MB, disk 354.0 MB, gpus: 0.0, runtime 101.634198 processing task id 251 item 4f7bffd9 with 93000 events completed on d12chas584.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.731, memory: 1146.0 MB, disk 355.0 MB, gpus: 0.0, runtime 96.270666 processing task id 229 item ebdcddd9 with 94500 events completed on d12chas577.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.622, memory: 1113.0 MB, disk 354.0 MB, gpus: 0.0, runtime 109.42431 submitted processing task id 396 item f765e3fa, with 117615 events submitted processing task id 397 item 77e7bf20, with 119579 events submitted processing task id 398 item 8b2a2c8c, with 119578 events submitted processing task id 399 item 311a4535, with 119263 events submitted processing task id 400 item e85b6676, with 119263 events submitted processing task id 401 item 909389a4, with 107933 events submitted processing task id 402 item 5f4f7785, with 107933 events submitted processing task id 403 item c52427a6, with 119035 events submitted processing task id 404 item 96d6a8ff, with 119035 events submitted processing task id 405 item fef67bc0, with 112931 events processing task id 280 item df2ee331 with 94500 events completed on qa-rtx6k-021.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.631, memory: 1162.0 MB, disk 354.0 MB, gpus: 0.0, runtime 85.448475 submitted processing task id 406 item 9006fc01, with 112931 events submitted processing task id 407 item 1369a390, with 112521 events submitted processing task id 408 item 2ed8cb5b, with 112521 events processing task id 198 item 6ee65d27 with 94500 events completed on d12chas580.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.577, memory: 1928.0 MB, disk 355.0 MB, gpus: 0.0, runtime 127.384451 submitted processing task id 409 item f6cc5187, with 109272 events submitted processing task id 410 item e4a9c968, with 109272 events submitted processing task id 411 item 22d5ed52, with 116061 events submitted processing task id 412 item ca160c19, with 116061 events processing task id 241 item d97ed809 with 94500 events completed on d12chas546.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.684, memory: 1089.0 MB, disk 355.0 MB, gpus: 0.0, runtime 106.248044 processing task id 252 item 7c67d602 with 94500 events completed on d12chas584.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.706, memory: 1086.0 MB, disk 354.0 MB, gpus: 0.0, runtime 101.512262 processing task id 253 item 22401c7d with 94500 events completed on d12chas584.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.679, memory: 2177.0 MB, disk 355.0 MB, gpus: 0.0, runtime 101.921759 processing task id 267 item 780bd2de with 87000 events completed on d12chas580.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.745, memory: 1873.0 MB, disk 354.0 MB, gpus: 0.0, runtime 91.314537 processing task id 266 item 56aa81bf with 87000 events completed on d12chas580.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.749, memory: 1181.0 MB, disk 353.0 MB, gpus: 0.0, runtime 93.022663 submitted processing task id 413 item e1aa23c1, with 118915 events submitted processing task id 414 item eed4e174, with 118914 events submitted processing task id 415 item 292d5022, with 83815 events submitted processing task id 416 item 0f734f89, with 83815 events submitted processing task id 417 item 3c3239aa, with 83815 events submitted processing task id 418 item cfec7462, with 83379 events submitted processing task id 419 item d11433bf, with 83379 events submitted processing task id 420 item 99de9cc3, with 83377 events submitted processing task id 421 item f2c2b43a, with 113449 events submitted processing task id 422 item 81e201b9, with 113449 events processing task id 255 item cdf636af with 94500 events completed on d12chas584.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.683, memory: 1112.0 MB, disk 354.0 MB, gpus: 0.0, runtime 104.5374 submitted processing task id 423 item ae0aef7c, with 117193 events submitted processing task id 424 item b0a54562, with 117193 events submitted processing task id 425 item 6430ab5b, with 120609 events submitted processing task id 426 item cf5b0133, with 120608 events submitted processing task id 427 item 4f82ab46, with 119308 events submitted processing task id 428 item 6e4f5411, with 119307 events submitted processing task id 429 item e98a8d57, with 83616 events submitted processing task id 430 item cba5cfef, with 83616 events processing task id 199 item 4a3a7388 with 94500 events completed on d12chas580.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.58, memory: 1119.0 MB, disk 355.0 MB, gpus: 0.0, runtime 131.535877 submitted processing task id 431 item b860bd22, with 83614 events processing task id 243 item 51688f84 with 93500 events completed on d12chas546.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.709, memory: 1231.0 MB, disk 354.0 MB, gpus: 0.0, runtime 110.495486 processing task id 249 item a5a933ab with 94500 events completed on d12chas545.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.641, memory: 2180.0 MB, disk 354.0 MB, gpus: 0.0, runtime 106.757787 submitted processing task id 432 item 646abe8b, with 83499 events submitted processing task id 433 item cfe62b91, with 83499 events submitted processing task id 434 item 11dc2e66, with 83499 events submitted processing task id 435 item 71d1f727, with 83494 events submitted processing task id 436 item 082610bb, with 83494 events processing task id 291 item 6a1c4b1f with 87500 events completed on d12chas546.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.791, memory: 2045.0 MB, disk 354.0 MB, gpus: 0.0, runtime 87.793822 submitted processing task id 437 item 1a530202, with 83494 events submitted processing task id 438 item d5beb3df, with 83835 events submitted processing task id 439 item fbbc3171, with 83835 events submitted processing task id 440 item 49bf6c0d, with 83833 events processing task id 268 item 7dbbcea3 with 86500 events completed on d12chas580.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.691, memory: 1065.0 MB, disk 353.0 MB, gpus: 0.0, runtime 99.105647 submitted processing task id 441 item b04b3218, with 83695 events submitted processing task id 442 item 2a311b79, with 83695 events submitted processing task id 443 item ec90fb76, with 83695 events submitted processing task id 444 item 34c2dd95, with 103085 events processing task id 283 item 61dd0dc1 with 86750 events completed on d12chas551.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.698, memory: 1065.0 MB, disk 354.0 MB, gpus: 0.0, runtime 94.14265 processing task id 214 item d873aa4f with 94500 events completed on d12chas548.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.576, memory: 1110.0 MB, disk 355.0 MB, gpus: 0.0, runtime 129.817011 processing task id 259 item 10b7f425 with 91250 events completed on d12chas564.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.666, memory: 1084.0 MB, disk 354.0 MB, gpus: 0.0, runtime 105.032011 processing task id 269 item 6ba551a9 with 86500 events completed on d12chas580.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.731, memory: 1088.0 MB, disk 354.0 MB, gpus: 0.0, runtime 100.053671 processing task id 288 item 7ee8cf1f with 85500 events completed on d12chas564.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.761, memory: 1059.0 MB, disk 353.0 MB, gpus: 0.0, runtime 90.291391 processing task id 248 item e670ff3d with 94500 events completed on d12chas545.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.632, memory: 1161.0 MB, disk 355.0 MB, gpus: 0.0, runtime 111.278406 submitted processing task id 445 item e6f01087, with 103084 events submitted processing task id 446 item 6b31a190, with 117814 events submitted processing task id 447 item f8fdfac1, with 117814 events submitted processing task id 448 item 8e8179e8, with 124612 events submitted processing task id 449 item 6d20edce, with 124612 events submitted processing task id 450 item 4dbb2b4e, with 123774 events submitted processing task id 451 item 5e3d8b77, with 123773 events submitted processing task id 452 item 2d531ad7, with 114922 events submitted processing task id 453 item bc5203f4, with 114922 events submitted processing task id 454 item 513cd0fb, with 83657 events processing task id 271 item 85ed4354 with 94500 events completed on d12chas553.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.68, memory: 2161.0 MB, disk 354.0 MB, gpus: 0.0, runtime 101.923052 submitted processing task id 455 item 3877ac8c, with 83657 events submitted processing task id 456 item e863d1d0, with 83657 events submitted processing task id 457 item 66b5e67f, with 83670 events submitted processing task id 458 item d9b24637, with 83670 events submitted processing task id 459 item 7a97b475, with 83669 events submitted processing task id 460 item 6a32324e, with 119759 events submitted processing task id 461 item b6d356f3, with 119758 events submitted processing task id 462 item 847493d1, with 83776 events submitted processing task id 463 item c2543901, with 83776 events processing task id 258 item fba22e1c with 91250 events completed on d12chas564.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.625, memory: 2084.0 MB, disk 354.0 MB, gpus: 0.0, runtime 107.171923 processing task id 215 item 9a2265fb with 94500 events completed on d12chas548.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.573, memory: 1162.0 MB, disk 355.0 MB, gpus: 0.0, runtime 132.669667 submitted accumulation task id 464 item accum_2, with 2298750 events processing task id 281 item 2404266e with 94500 events completed on d12chas551.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.702, memory: 1115.0 MB, disk 354.0 MB, gpus: 0.0, runtime 98.593613 submitted processing task id 465 item 8829337a, with 83776 events submitted processing task id 466 item bcea9e95, with 109535 events submitted processing task id 467 item afeb9b08, with 109534 events submitted processing task id 468 item f4920729, with 109131 events submitted processing task id 469 item f38ff8ed, with 109130 events submitted processing task id 470 item 52f122d1, with 110778 events submitted processing task id 471 item b57bb24a, with 110778 events submitted processing task id 472 item 62891c70, with 117734 events accumulating task id 395 item accum_1 with 2331000 events completed on d12chas546.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.998, memory: 608.0 MB, disk 1012.0 MB, gpus: 0.0, runtime 18.061631 submitted processing task id 473 item e335319a, with 117734 events submitted processing task id 474 item 937a33eb, with 118057 events submitted processing task id 475 item 46292915, with 118056 events submitted processing task id 476 item 9abfe290, with 118018 events processing task id 289 item a4c954ce with 85500 events completed on d12chas564.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.722, memory: 1067.0 MB, disk 353.0 MB, gpus: 0.0, runtime 100.223935 submitted processing task id 477 item 3809c2f8, with 118017 events processing task id 286 item 6a687e18 with 94500 events completed on d12chas564.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.725, memory: 2093.0 MB, disk 355.0 MB, gpus: 0.0, runtime 102.860831 submitted processing task id 478 item 7c4db65b, with 117961 events submitted processing task id 479 item 1d227c07, with 117960 events submitted processing task id 480 item a1b52282, with 79917 events submitted processing task id 481 item b6954b26, with 106863 events processing task id 324 item f7696514 with 27000 events completed on d12chas548.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.779, memory: 568.0 MB, disk 343.0 MB, gpus: 0.0, runtime 45.25675 submitted processing task id 482 item 2c090759, with 106862 events submitted processing task id 483 item a718850f, with 115140 events accumulating task id 464 item accum_2 with 2298750 events completed on d12chas548.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.998, memory: 549.0 MB, disk 1007.0 MB, gpus: 0.0, runtime 14.639729 processing task id 294 item 4b082725 with 94500 events completed on d12chas546.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.683, memory: 1080.0 MB, disk 354.0 MB, gpus: 0.0, runtime 108.4429 processing task id 295 item 5b25d527 with 94500 events completed on d12chas546.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.656, memory: 1112.0 MB, disk 354.0 MB, gpus: 0.0, runtime 109.275509 submitted processing task id 484 item d628953f, with 115140 events submitted processing task id 485 item 353e4b0a, with 117970 events submitted processing task id 486 item 47c87f57, with 117969 events submitted processing task id 487 item 7b0b29c5, with 118118 events submitted processing task id 488 item 76d4213f, with 118117 events submitted processing task id 489 item 92b263e7, with 114381 events processing task id 309 item 45204ecc with 94500 events completed on d12chas547.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.042, memory: 134.0 MB, disk 330.0 MB, gpus: 0.0, runtime 62.998247 output: Using /cvmfs certificates. 2021-10-02 22:46:45 python_package_run(2552): setting up conda environment at /var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:46:45 python_package_run(2552): waiting for lock on directory /var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:46:45 python_package_run(2552): directory /var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:46:45 python_package_run(2552): activating environment 2021-10-02 22:46:45 python_package_run(2552): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_8819/worker-230547-8824/t.309/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1135, in _work_function factory = NanoEventsFactory.from_root( File "/var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/nanoevents/factory.py", line 97, in from_root tree = file[treepath] File "/var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 2081, in __getitem__ return self.key(where).get() File "/var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 2478, in get chunk, cursor = self.get_uncompressed_chunk_cursor() File "/var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 2530, in get_uncompressed_chunk_cursor chunk = self._file.chunk(data_start, data_stop) File "/var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 1207, in chunk return self._source.chunk(start, stop) File "/var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 305, in chunk data = self._resource.get(start, stop) File "/var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 168, in get self._xrd_error(status) File "/var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Socket timeout in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_220.root 2021-10-02 22:47:48 python_package_run(2552): command line exit code: 1 2021-10-02 22:47:48 python_package_run(2552): total runtime: 62.888 seconds task id 309 item 45204ecc failed: OUTPUT_MISS ('UL17_tttt_b4', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_220.root', 'Events', 94500, 189000) 45204ecc without result. resubmitting 45204ecc partly as 45204ecc. 1 attempt(s) left. processing task id 297 item e442d6ab with 94500 events completed on qa-rtx6k-021.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.564, memory: 1995.0 MB, disk 355.0 MB, gpus: 0.0, runtime 97.652146 processing task id 325 item 950c1f9b with 85500 events completed on d12chas548.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.686, memory: 1059.0 MB, disk 354.0 MB, gpus: 0.0, runtime 95.992088 submitted processing task id 491 item c18ecd8e, with 114381 events processing task id 322 item 58262c43 with 86750 events completed on d12chas552.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.668, memory: 2067.0 MB, disk 354.0 MB, gpus: 0.0, runtime 96.189667 submitted processing task id 492 item dbd84d3a, with 117841 events submitted processing task id 493 item 4f4fcba7, with 117840 events processing task id 326 item a2d35269 with 85500 events completed on d12chas548.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.684, memory: 1055.0 MB, disk 353.0 MB, gpus: 0.0, runtime 101.660166 processing task id 333 item 23498b27 with 89750 events completed on d12chas582.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.629, memory: 2014.0 MB, disk 354.0 MB, gpus: 0.0, runtime 101.327316 processing task id 334 item e9b53f28 with 89750 events completed on d12chas545.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.635, memory: 1070.0 MB, disk 354.0 MB, gpus: 0.0, runtime 103.41044 submitted processing task id 494 item a4d1e7a1, with 113673 events submitted processing task id 495 item 66758e11, with 113672 events processing task id 305 item 7dd0d1a7 with 92750 events completed on d12chas553.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.647, memory: 2141.0 MB, disk 355.0 MB, gpus: 0.0, runtime 117.38696 submitted processing task id 496 item 41a614f5, with 111634 events processing task id 328 item df24000d with 86000 events completed on d12chas548.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.614, memory: 1083.0 MB, disk 353.0 MB, gpus: 0.0, runtime 108.269843 submitted processing task id 497 item 7f44a48f, with 111633 events processing task id 310 item c13c0a36 with 94500 events completed on d12chas547.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.654, memory: 2161.0 MB, disk 354.0 MB, gpus: 0.0, runtime 112.156628 submitted processing task id 498 item c36d2231, with 107092 events processing task id 318 item c3ca71c8 with 94500 events completed on d12chas553.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.66, memory: 1094.0 MB, disk 355.0 MB, gpus: 0.0, runtime 112.949123 submitted processing task id 499 item a5e80bc0, with 107091 events processing task id 311 item 6160f7b1 with 94500 events completed on d12chas547.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.636, memory: 2180.0 MB, disk 354.0 MB, gpus: 0.0, runtime 116.449548 submitted processing task id 500 item 8f01a6db, with 111956 events submitted processing task id 501 item 5085a027, with 111956 events processing task id 299 item ebfa1e03 with 94500 events completed on qa-rtx6k-021.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.441, memory: 1126.0 MB, disk 354.0 MB, gpus: 0.0, runtime 128.212469 processing task id 335 item 154b7686 with 94500 events completed on d12chas584.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.56, memory: 1161.0 MB, disk 354.0 MB, gpus: 0.0, runtime 126.533738 submitted processing task id 502 item 0ba00551, with 107703 events processing task id 298 item a43a86e9 with 94500 events completed on qa-rtx6k-021.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.442, memory: 2069.0 MB, disk 354.0 MB, gpus: 0.0, runtime 133.503777 submitted processing task id 503 item 6894b4b3, with 107702 events processing task id 336 item eec901e9 with 94500 events completed on d12chas553.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.57, memory: 1111.0 MB, disk 354.0 MB, gpus: 0.0, runtime 136.032089 processing task id 284 item b1fde89d with 94500 events completed on d12chas551.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.355, memory: 1112.0 MB, disk 355.0 MB, gpus: 0.0, runtime 199.844079 processing task id 285 item a00c4e65 with 94500 events completed on d12chas551.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.356, memory: 1110.0 MB, disk 354.0 MB, gpus: 0.0, runtime 201.105347 submitted accumulation task id 504 item accum_3, with 6681250 events submitted processing task id 505 item 57fb30c9, with 111196 events processing task id 306 item 159cc5c9 with 87000 events completed on qa-rtx6k-021.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.357, memory: 2072.0 MB, disk 354.0 MB, gpus: 0.0, runtime 145.572645 processing task id 300 item 0deaad7a with 85250 events completed on qa-rtx6k-021.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.361, memory: 2042.0 MB, disk 354.0 MB, gpus: 0.0, runtime 146.322657 submitted processing task id 506 item 25cfe105, with 111195 events submitted processing task id 507 item 450d61a2, with 112585 events submitted processing task id 508 item 108cb9b2, with 112585 events submitted processing task id 509 item c6ad0ecc, with 111303 events submitted processing task id 510 item af1c16a1, with 111302 events processing task id 307 item 48271323 with 87000 events completed on qa-rtx6k-021.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.351, memory: 1037.0 MB, disk 353.0 MB, gpus: 0.0, runtime 149.818481 submitted processing task id 511 item e15053f0, with 113889 events submitted processing task id 512 item 28cc63de, with 113888 events accumulating task id 504 item accum_3 with 6681250 events completed on d12chas551.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.998, memory: 684.0 MB, disk 1101.0 MB, gpus: 0.0, runtime 14.983074 processing task id 323 item 1aff935a with 86750 events completed on qa-rtx6k-021.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.33, memory: 2090.0 MB, disk 354.0 MB, gpus: 0.0, runtime 155.466788 processing task id 296 item 10d36d02 with 94500 events completed on qa-rtx6k-021.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.348, memory: 1315.0 MB, disk 354.0 MB, gpus: 0.0, runtime 162.790662 submitted processing task id 513 item 61d7ae8d, with 107507 events submitted processing task id 514 item c596506b, with 107506 events processing task id 302 item b0cf619e with 85250 events completed on d12chas547.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.39, memory: 1094.0 MB, disk 354.0 MB, gpus: 0.0, runtime 162.841436 processing task id 327 item 2b508358 with 86000 events completed on d12chas548.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.411, memory: 1085.0 MB, disk 354.0 MB, gpus: 0.0, runtime 159.202505 processing task id 301 item dcd605ae with 85250 events completed on d12chas587.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.404, memory: 1059.0 MB, disk 354.0 MB, gpus: 0.0, runtime 164.157424 submitted processing task id 515 item 6d077bb2, with 84167 events submitted processing task id 516 item e09a20c8, with 84167 events submitted processing task id 517 item 485f2cd3, with 84166 events processing task id 303 item e0f9c340 with 85250 events completed on d12chas548.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.4, memory: 1053.0 MB, disk 353.0 MB, gpus: 0.0, runtime 164.348122 processing task id 315 item 56830adb with 85500 events completed on d12chas553.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.413, memory: 1055.0 MB, disk 354.0 MB, gpus: 0.0, runtime 164.231008 submitted processing task id 518 item cea5f55f, with 122000 events submitted processing task id 519 item bfac42b9, with 122000 events processing task id 304 item 97aa439d with 92750 events completed on d12chas580.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.395, memory: 1083.0 MB, disk 354.0 MB, gpus: 0.0, runtime 171.343113 submitted processing task id 520 item 84c63233, with 84000 events processing task id 320 item c7abad5d with 94500 events completed on d12chas546.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.411, memory: 1112.0 MB, disk 354.0 MB, gpus: 0.0, runtime 169.005517 submitted processing task id 521 item ce1334d4, with 84000 events processing task id 319 item 859a646b with 94500 events completed on d12chas589.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.401, memory: 1118.0 MB, disk 355.0 MB, gpus: 0.0, runtime 170.393817 submitted processing task id 522 item ed0f0f1d, with 84000 events processing task id 317 item e3e82e83 with 87500 events completed on d12chas553.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.417, memory: 1014.0 MB, disk 354.0 MB, gpus: 0.0, runtime 172.337741 submitted processing task id 523 item f5d0969e, with 114750 events processing task id 314 item 47daae98 with 85500 events completed on d12chas553.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.425, memory: 2036.0 MB, disk 353.0 MB, gpus: 0.0, runtime 174.361406 submitted processing task id 524 item 96480b02, with 114750 events processing task id 337 item cdf22113 with 89000 events completed on d12chas553.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.414, memory: 1110.0 MB, disk 354.0 MB, gpus: 0.0, runtime 174.216378 submitted processing task id 525 item fb4260f6, with 123250 events processing task id 331 item 33e0e901 with 93750 events completed on d12chas553.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.41, memory: 1154.0 MB, disk 354.0 MB, gpus: 0.0, runtime 174.728921 submitted processing task id 526 item 5e7035ac, with 123250 events processing task id 321 item ca251bc6 with 94500 events completed on d12chas577.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.421, memory: 1112.0 MB, disk 355.0 MB, gpus: 0.0, runtime 176.196534 submitted processing task id 527 item be614941, with 84334 events processing task id 332 item 31c18953 with 93750 events completed on d12chas564.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.422, memory: 1155.0 MB, disk 354.0 MB, gpus: 0.0, runtime 176.769008 submitted processing task id 528 item 1fdd31e5, with 84334 events submitted processing task id 529 item eed89c36, with 84332 events processing task id 308 item ed569bb6 with 94500 events completed on d12chas553.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.421, memory: 1108.0 MB, disk 355.0 MB, gpus: 0.0, runtime 186.20085 submitted processing task id 530 item 3c0912a7, with 117250 events processing task id 329 item 8a469fac with 87000 events completed on d12chas551.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.333, memory: 1094.0 MB, disk 354.0 MB, gpus: 0.0, runtime 194.538234 processing task id 330 item df8ada55 with 87000 events completed on d12chas580.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.327, memory: 1092.0 MB, disk 354.0 MB, gpus: 0.0, runtime 196.231487 submitted processing task id 531 item 9b6c849e, with 117250 events submitted processing task id 532 item e44ecb58, with 84000 events processing task id 312 item 97c4bb33 with 94500 events completed on d12chas547.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.34, memory: 1112.0 MB, disk 354.0 MB, gpus: 0.0, runtime 205.272234 processing task id 313 item d11cd9ba with 94500 events completed on d12chas547.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.349, memory: 1112.0 MB, disk 354.0 MB, gpus: 0.0, runtime 206.652474 submitted accumulation task id 533 item accum_4, with 8832250 events submitted processing task id 534 item bee9251a, with 84000 events accumulating task id 533 item accum_4 with 8832250 events completed on d12chas547.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.998, memory: 547.0 MB, disk 1080.0 MB, gpus: 0.0, runtime 14.960186 submitted processing task id 535 item 8e6250aa, with 84000 events processing task id 343 item 3e4ad448 with 94500 events completed on qa-rtx6k-021.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.007, memory: 133.0 MB, disk 330.0 MB, gpus: 0.0, runtime 302.227788 output: Using /cvmfs certificates. 2021-10-02 22:46:48 python_package_run(159239): setting up conda environment at /var/condor/execute/dir_155375/worker-230547-155384/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:46:48 python_package_run(159239): waiting for lock on directory /var/condor/execute/dir_155375/worker-230547-155384/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:46:49 python_package_run(159239): directory /var/condor/execute/dir_155375/worker-230547-155384/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:46:49 python_package_run(159239): activating environment 2021-10-02 22:46:49 python_package_run(159239): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_155375/worker-230547-155384/t.343/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_155375/worker-230547-155384/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_155375/worker-230547-155384/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_155375/worker-230547-155384/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_155375/worker-230547-155384/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_155375/worker-230547-155384/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_155375/worker-230547-155384/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_155375/worker-230547-155384/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_155375/worker-230547-155384/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_10092.root 2021-10-02 22:51:51 python_package_run(159239): command line exit code: 1 2021-10-02 22:51:51 python_package_run(159239): total runtime: 302.128 seconds task id 343 item 3e4ad448 failed: OUTPUT_MISS ('UL17_tttt_b4', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_10092.root', 'Events', 0, 94500) 3e4ad448 without result. resubmitting 3e4ad448 partly as 3e4ad448. 1 attempt(s) left. processing task id 348 item 5fcba9fa with 94250 events completed on d12chas580.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 140.0 MB, disk 330.0 MB, gpus: 0.0, runtime 302.985675 output: Using /cvmfs certificates. 2021-10-02 22:46:48 python_package_run(8214): setting up conda environment at /var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:46:48 python_package_run(8214): waiting for lock on directory /var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:46:49 python_package_run(8214): directory /var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:46:49 python_package_run(8214): activating environment 2021-10-02 22:46:49 python_package_run(8214): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_5228/worker-230547-5303/t.348/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_10090.root 2021-10-02 22:51:51 python_package_run(8214): command line exit code: 1 2021-10-02 22:51:51 python_package_run(8214): total runtime: 302.874 seconds task id 348 item 5fcba9fa failed: OUTPUT_MISS ('UL17_tttt_b4', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_10090.root', 'Events', 94250, 188500) 5fcba9fa without result. resubmitting 5fcba9fa partly as 5fcba9fa. 1 attempt(s) left. processing task id 347 item 3e008e90 with 94250 events completed on d12chas580.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 140.0 MB, disk 330.0 MB, gpus: 0.0, runtime 303.070069 output: Using /cvmfs certificates. 2021-10-02 22:46:48 python_package_run(8215): setting up conda environment at /var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:46:48 python_package_run(8215): waiting for lock on directory /var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:46:49 python_package_run(8215): directory /var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:46:49 python_package_run(8215): activating environment 2021-10-02 22:46:49 python_package_run(8215): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_5228/worker-230547-5303/t.347/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_10090.root 2021-10-02 22:51:51 python_package_run(8215): command line exit code: 1 2021-10-02 22:51:51 python_package_run(8215): total runtime: 302.959 seconds task id 347 item 3e008e90 failed: OUTPUT_MISS ('UL17_tttt_b4', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_10090.root', 'Events', 0, 94250) 3e008e90 without result. resubmitting 3e008e90 partly as 3e008e90. 1 attempt(s) left. processing task id 344 item d4f5ccfc with 94500 events completed on d12chas580.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 138.0 MB, disk 330.0 MB, gpus: 0.0, runtime 303.128773 output: Using /cvmfs certificates. 2021-10-02 22:46:48 python_package_run(8218): setting up conda environment at /var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:46:48 python_package_run(8218): waiting for lock on directory /var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:46:49 python_package_run(8218): directory /var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:46:49 python_package_run(8218): activating environment 2021-10-02 22:46:49 python_package_run(8218): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_5228/worker-230547-5303/t.344/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_10092.root 2021-10-02 22:51:51 python_package_run(8218): command line exit code: 1 2021-10-02 22:51:51 python_package_run(8218): total runtime: 303.020 seconds task id 344 item d4f5ccfc failed: OUTPUT_MISS ('UL17_tttt_b4', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_10092.root', 'Events', 94500, 189000) d4f5ccfc without result. resubmitting d4f5ccfc partly as d4f5ccfc. 1 attempt(s) left. processing task id 346 item 27aaaee0 with 94500 events completed on d12chas580.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 134.0 MB, disk 330.0 MB, gpus: 0.0, runtime 303.194167 output: Using /cvmfs certificates. 2021-10-02 22:46:48 python_package_run(8216): setting up conda environment at /var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:46:48 python_package_run(8216): waiting for lock on directory /var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:46:49 python_package_run(8216): directory /var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:46:49 python_package_run(8216): activating environment 2021-10-02 22:46:49 python_package_run(8216): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_5228/worker-230547-5303/t.346/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_10091.root 2021-10-02 22:51:52 python_package_run(8216): command line exit code: 1 2021-10-02 22:51:52 python_package_run(8216): total runtime: 303.080 seconds task id 346 item 27aaaee0 failed: OUTPUT_MISS ('UL17_tttt_b4', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_10091.root', 'Events', 94500, 189000) 27aaaee0 without result. resubmitting 27aaaee0 partly as 27aaaee0. 1 attempt(s) left. processing task id 345 item bfa76817 with 94500 events completed on d12chas580.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 133.0 MB, disk 330.0 MB, gpus: 0.0, runtime 303.228375 output: Using /cvmfs certificates. 2021-10-02 22:46:48 python_package_run(8217): setting up conda environment at /var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:46:48 python_package_run(8217): waiting for lock on directory /var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:46:49 python_package_run(8217): directory /var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:46:49 python_package_run(8217): activating environment 2021-10-02 22:46:49 python_package_run(8217): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_5228/worker-230547-5303/t.345/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_10091.root 2021-10-02 22:51:52 python_package_run(8217): command line exit code: 1 2021-10-02 22:51:52 python_package_run(8217): total runtime: 303.126 seconds task id 345 item bfa76817 failed: OUTPUT_MISS ('UL17_tttt_b4', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_10091.root', 'Events', 0, 94500) bfa76817 without result. resubmitting bfa76817 partly as bfa76817. 1 attempt(s) left. processing task id 341 item 7d52b268 with 94500 events completed on d12chas564.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 133.0 MB, disk 330.0 MB, gpus: 0.0, runtime 303.537816 output: Using /cvmfs certificates. 2021-10-02 22:46:48 python_package_run(25634): setting up conda environment at /var/condor/execute/dir_23236/worker-230547-23258/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:46:48 python_package_run(25634): waiting for lock on directory /var/condor/execute/dir_23236/worker-230547-23258/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:46:49 python_package_run(25634): directory /var/condor/execute/dir_23236/worker-230547-23258/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:46:49 python_package_run(25634): activating environment 2021-10-02 22:46:49 python_package_run(25634): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_23236/worker-230547-23258/t.341/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_23236/worker-230547-23258/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_23236/worker-230547-23258/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_23236/worker-230547-23258/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_23236/worker-230547-23258/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_23236/worker-230547-23258/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_23236/worker-230547-23258/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_23236/worker-230547-23258/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_23236/worker-230547-23258/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_10093.root 2021-10-02 22:51:52 python_package_run(25634): command line exit code: 1 2021-10-02 22:51:52 python_package_run(25634): total runtime: 303.421 seconds task id 341 item 7d52b268 failed: OUTPUT_MISS ('UL17_tttt_b4', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_10093.root', 'Events', 0, 94500) 7d52b268 without result. resubmitting 7d52b268 partly as 7d52b268. 1 attempt(s) left. processing task id 352 item 34a284aa with 94500 events completed on d12chas547.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 134.0 MB, disk 330.0 MB, gpus: 0.0, runtime 302.874508 output: Using /cvmfs certificates. 2021-10-02 22:46:51 python_package_run(2837): setting up conda environment at /var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:46:51 python_package_run(2837): waiting for lock on directory /var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:46:51 python_package_run(2837): directory /var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:46:51 python_package_run(2837): activating environment 2021-10-02 22:46:51 python_package_run(2837): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_8819/worker-230547-8824/t.352/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_10088.root 2021-10-02 22:51:54 python_package_run(2837): command line exit code: 1 2021-10-02 22:51:54 python_package_run(2837): total runtime: 302.764 seconds task id 352 item 34a284aa failed: OUTPUT_MISS ('UL17_tttt_b4', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_10088.root', 'Events', 94500, 189000) 34a284aa without result. resubmitting 34a284aa partly as 34a284aa. 1 attempt(s) left. processing task id 355 item 0e2e8207 with 94500 events completed on d12chas580.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 134.0 MB, disk 330.0 MB, gpus: 0.0, runtime 302.896102 output: Using /cvmfs certificates. 2021-10-02 22:46:51 python_package_run(8599): setting up conda environment at /var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:46:51 python_package_run(8599): waiting for lock on directory /var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:46:51 python_package_run(8599): directory /var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:46:51 python_package_run(8599): activating environment 2021-10-02 22:46:51 python_package_run(8599): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_5228/worker-230547-5303/t.355/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_10086.root 2021-10-02 22:51:54 python_package_run(8599): command line exit code: 1 2021-10-02 22:51:54 python_package_run(8599): total runtime: 302.787 seconds task id 355 item 0e2e8207 failed: OUTPUT_MISS ('UL17_tttt_b4', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_10086.root', 'Events', 0, 94500) 0e2e8207 without result. resubmitting 0e2e8207 partly as 0e2e8207. 1 attempt(s) left. processing task id 354 item e5241d59 with 94500 events completed on d12chas580.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.009, memory: 136.0 MB, disk 330.0 MB, gpus: 0.0, runtime 302.96426 output: Using /cvmfs certificates. 2021-10-02 22:46:51 python_package_run(8600): setting up conda environment at /var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:46:51 python_package_run(8600): waiting for lock on directory /var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:46:51 python_package_run(8600): directory /var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:46:51 python_package_run(8600): activating environment 2021-10-02 22:46:51 python_package_run(8600): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_5228/worker-230547-5303/t.354/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_10087.root 2021-10-02 22:51:54 python_package_run(8600): command line exit code: 1 2021-10-02 22:51:54 python_package_run(8600): total runtime: 302.852 seconds task id 354 item e5241d59 failed: OUTPUT_MISS ('UL17_tttt_b4', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_10087.root', 'Events', 94500, 189000) e5241d59 without result. resubmitting e5241d59 partly as e5241d59. 1 attempt(s) left. processing task id 356 item 01a58bfd with 94500 events completed on d12chas580.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 136.0 MB, disk 330.0 MB, gpus: 0.0, runtime 303.157896 output: Using /cvmfs certificates. 2021-10-02 22:46:51 python_package_run(8598): setting up conda environment at /var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:46:51 python_package_run(8598): waiting for lock on directory /var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:46:51 python_package_run(8598): directory /var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:46:51 python_package_run(8598): activating environment 2021-10-02 22:46:51 python_package_run(8598): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_5228/worker-230547-5303/t.356/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_10086.root 2021-10-02 22:51:54 python_package_run(8598): command line exit code: 1 2021-10-02 22:51:54 python_package_run(8598): total runtime: 303.049 seconds task id 356 item 01a58bfd failed: OUTPUT_MISS ('UL17_tttt_b4', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_10086.root', 'Events', 94500, 189000) 01a58bfd without result. resubmitting 01a58bfd partly as 01a58bfd. 1 attempt(s) left. processing task id 353 item 056212f9 with 94500 events completed on d12chas580.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 140.0 MB, disk 330.0 MB, gpus: 0.0, runtime 303.226871 output: Using /cvmfs certificates. 2021-10-02 22:46:51 python_package_run(8601): setting up conda environment at /var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:46:51 python_package_run(8601): waiting for lock on directory /var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:46:51 python_package_run(8601): directory /var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:46:51 python_package_run(8601): activating environment 2021-10-02 22:46:51 python_package_run(8601): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_5228/worker-230547-5303/t.353/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_10087.root 2021-10-02 22:51:54 python_package_run(8601): command line exit code: 1 2021-10-02 22:51:54 python_package_run(8601): total runtime: 303.117 seconds task id 353 item 056212f9 failed: OUTPUT_MISS ('UL17_tttt_b4', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_10087.root', 'Events', 0, 94500) 056212f9 without result. resubmitting 056212f9 partly as 056212f9. 1 attempt(s) left. processing task id 350 item 43010f60 with 94500 events completed on d12chas547.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 133.0 MB, disk 330.0 MB, gpus: 0.0, runtime 303.299487 output: Using /cvmfs certificates. 2021-10-02 22:46:51 python_package_run(2839): setting up conda environment at /var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:46:51 python_package_run(2839): waiting for lock on directory /var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:46:51 python_package_run(2839): directory /var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:46:51 python_package_run(2839): activating environment 2021-10-02 22:46:51 python_package_run(2839): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_8819/worker-230547-8824/t.350/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_10089.root 2021-10-02 22:51:54 python_package_run(2839): command line exit code: 1 2021-10-02 22:51:54 python_package_run(2839): total runtime: 303.183 seconds task id 350 item 43010f60 failed: OUTPUT_MISS ('UL17_tttt_b4', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_10089.root', 'Events', 94500, 189000) 43010f60 without result. resubmitting 43010f60 partly as 43010f60. 1 attempt(s) left. processing task id 363 item ba70ab41 with 113080 events completed on d12chas547.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 134.0 MB, disk 330.0 MB, gpus: 0.0, runtime 302.656104 output: Using /cvmfs certificates. 2021-10-02 22:46:52 python_package_run(2983): setting up conda environment at /var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:46:52 python_package_run(2983): waiting for lock on directory /var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:46:52 python_package_run(2983): directory /var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:46:52 python_package_run(2983): activating environment 2021-10-02 22:46:52 python_package_run(2983): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_8819/worker-230547-8824/t.363/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttlnuJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_8152.root 2021-10-02 22:51:54 python_package_run(2983): command line exit code: 1 2021-10-02 22:51:54 python_package_run(2983): total runtime: 302.552 seconds task id 363 item ba70ab41 failed: OUTPUT_MISS ('UL17_ttlnuJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttlnuJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_8152.root', 'Events', 0, 113080) ba70ab41 without result. resubmitting ba70ab41 partly as ba70ab41. 1 attempt(s) left. processing task id 362 item 46e81f1b with 113392 events completed on d12chas587.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 134.0 MB, disk 330.0 MB, gpus: 0.0, runtime 302.82642 output: Using /cvmfs certificates. 2021-10-02 22:46:53 python_package_run(31315): setting up conda environment at /var/condor/execute/dir_29058/worker-230547-29063/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:46:53 python_package_run(31315): waiting for lock on directory /var/condor/execute/dir_29058/worker-230547-29063/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:46:53 python_package_run(31315): directory /var/condor/execute/dir_29058/worker-230547-29063/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:46:53 python_package_run(31315): activating environment 2021-10-02 22:46:53 python_package_run(31315): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_29058/worker-230547-29063/t.362/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_29058/worker-230547-29063/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_29058/worker-230547-29063/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_29058/worker-230547-29063/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_29058/worker-230547-29063/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_29058/worker-230547-29063/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_29058/worker-230547-29063/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_29058/worker-230547-29063/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_29058/worker-230547-29063/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttlnuJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_8793.root 2021-10-02 22:51:56 python_package_run(31315): command line exit code: 1 2021-10-02 22:51:56 python_package_run(31315): total runtime: 302.713 seconds task id 362 item 46e81f1b failed: OUTPUT_MISS ('UL17_ttlnuJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttlnuJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_8793.root', 'Events', 113392, 226784) 46e81f1b without result. resubmitting 46e81f1b partly as 46e81f1b. 1 attempt(s) left. processing task id 364 item 2da82cb2 with 113079 events completed on d12chas587.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 140.0 MB, disk 330.0 MB, gpus: 0.0, runtime 303.261881 output: Using /cvmfs certificates. 2021-10-02 22:46:53 python_package_run(31314): setting up conda environment at /var/condor/execute/dir_29058/worker-230547-29063/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:46:53 python_package_run(31314): waiting for lock on directory /var/condor/execute/dir_29058/worker-230547-29063/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:46:53 python_package_run(31314): directory /var/condor/execute/dir_29058/worker-230547-29063/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:46:53 python_package_run(31314): activating environment 2021-10-02 22:46:53 python_package_run(31314): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_29058/worker-230547-29063/t.364/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_29058/worker-230547-29063/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_29058/worker-230547-29063/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_29058/worker-230547-29063/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_29058/worker-230547-29063/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_29058/worker-230547-29063/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_29058/worker-230547-29063/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_29058/worker-230547-29063/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_29058/worker-230547-29063/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttlnuJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_8152.root 2021-10-02 22:51:56 python_package_run(31314): command line exit code: 1 2021-10-02 22:51:56 python_package_run(31314): total runtime: 303.145 seconds task id 364 item 2da82cb2 failed: OUTPUT_MISS ('UL17_ttlnuJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttlnuJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_8152.root', 'Events', 113080, 226159) 2da82cb2 without result. resubmitting 2da82cb2 partly as 2da82cb2. 1 attempt(s) left. processing task id 339 item 0b1bcd41 with 94250 events completed on d12chas577.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 134.0 MB, disk 330.0 MB, gpus: 0.0, runtime 302.048068 output: Using /cvmfs certificates. 2021-10-02 22:46:54 python_package_run(24963): setting up conda environment at /var/condor/execute/dir_22513/worker-230547-22528/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:46:54 python_package_run(24963): waiting for lock on directory /var/condor/execute/dir_22513/worker-230547-22528/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:46:54 python_package_run(24963): directory /var/condor/execute/dir_22513/worker-230547-22528/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:46:54 python_package_run(24963): activating environment 2021-10-02 22:46:54 python_package_run(24963): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_22513/worker-230547-22528/t.339/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_22513/worker-230547-22528/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_22513/worker-230547-22528/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_22513/worker-230547-22528/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_22513/worker-230547-22528/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_22513/worker-230547-22528/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_22513/worker-230547-22528/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_22513/worker-230547-22528/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_22513/worker-230547-22528/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_10095.root 2021-10-02 22:51:56 python_package_run(24963): command line exit code: 1 2021-10-02 22:51:56 python_package_run(24963): total runtime: 301.958 seconds task id 339 item 0b1bcd41 failed: OUTPUT_MISS ('UL17_tttt_b4', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_10095.root', 'Events', 0, 94250) 0b1bcd41 without result. resubmitting 0b1bcd41 partly as 0b1bcd41. 1 attempt(s) left. processing task id 359 item 044f2f1f with 119113 events completed on d12chas587.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.012, memory: 138.0 MB, disk 330.0 MB, gpus: 0.0, runtime 303.875759 output: Using /cvmfs certificates. 2021-10-02 22:46:53 python_package_run(31318): setting up conda environment at /var/condor/execute/dir_29058/worker-230547-29063/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:46:53 python_package_run(31318): waiting for lock on directory /var/condor/execute/dir_29058/worker-230547-29063/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:46:53 python_package_run(31318): directory /var/condor/execute/dir_29058/worker-230547-29063/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:46:53 python_package_run(31318): activating environment 2021-10-02 22:46:53 python_package_run(31318): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_29058/worker-230547-29063/t.359/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_29058/worker-230547-29063/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_29058/worker-230547-29063/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_29058/worker-230547-29063/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_29058/worker-230547-29063/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_29058/worker-230547-29063/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_29058/worker-230547-29063/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_29058/worker-230547-29063/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_29058/worker-230547-29063/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttlnuJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_9766.root 2021-10-02 22:51:57 python_package_run(31318): command line exit code: 1 2021-10-02 22:51:57 python_package_run(31318): total runtime: 303.758 seconds task id 359 item 044f2f1f failed: OUTPUT_MISS ('UL17_ttlnuJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttlnuJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_9766.root', 'Events', 0, 119113) 044f2f1f without result. resubmitting 044f2f1f partly as 044f2f1f. 1 attempt(s) left. processing task id 369 item 397c23ee with 112549 events completed on d12chas552.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 134.0 MB, disk 330.0 MB, gpus: 0.0, runtime 302.800255 output: Using /cvmfs certificates. 2021-10-02 22:46:54 python_package_run(11086): setting up conda environment at /var/condor/execute/dir_8823/worker-230547-8828/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:46:54 python_package_run(11086): waiting for lock on directory /var/condor/execute/dir_8823/worker-230547-8828/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:46:54 python_package_run(11086): directory /var/condor/execute/dir_8823/worker-230547-8828/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:46:54 python_package_run(11086): activating environment 2021-10-02 22:46:54 python_package_run(11086): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_8823/worker-230547-8828/t.369/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_8823/worker-230547-8828/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_8823/worker-230547-8828/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_8823/worker-230547-8828/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_8823/worker-230547-8828/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_8823/worker-230547-8828/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_8823/worker-230547-8828/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_8823/worker-230547-8828/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_8823/worker-230547-8828/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttlnuJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_6353.root 2021-10-02 22:51:57 python_package_run(11086): command line exit code: 1 2021-10-02 22:51:57 python_package_run(11086): total runtime: 302.679 seconds task id 369 item 397c23ee failed: OUTPUT_MISS ('UL17_ttlnuJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttlnuJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_6353.root', 'Events', 0, 112549) 397c23ee without result. resubmitting 397c23ee partly as 397c23ee. 1 attempt(s) left. processing task id 367 item ad705893 with 115080 events completed on d12chas552.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 134.0 MB, disk 330.0 MB, gpus: 0.0, runtime 302.944514 output: Using /cvmfs certificates. 2021-10-02 22:46:54 python_package_run(11091): setting up conda environment at /var/condor/execute/dir_8823/worker-230547-8828/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:46:54 python_package_run(11091): waiting for lock on directory /var/condor/execute/dir_8823/worker-230547-8828/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:46:55 python_package_run(11091): directory /var/condor/execute/dir_8823/worker-230547-8828/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:46:55 python_package_run(11091): activating environment 2021-10-02 22:46:55 python_package_run(11091): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_8823/worker-230547-8828/t.367/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_8823/worker-230547-8828/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_8823/worker-230547-8828/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_8823/worker-230547-8828/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_8823/worker-230547-8828/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_8823/worker-230547-8828/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_8823/worker-230547-8828/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_8823/worker-230547-8828/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_8823/worker-230547-8828/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttlnuJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_6942.root 2021-10-02 22:51:57 python_package_run(11091): command line exit code: 1 2021-10-02 22:51:57 python_package_run(11091): total runtime: 302.823 seconds task id 367 item ad705893 failed: OUTPUT_MISS ('UL17_ttlnuJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttlnuJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_6942.root', 'Events', 0, 115080) ad705893 without result. resubmitting ad705893 partly as ad705893. 1 attempt(s) left. processing task id 368 item 7434bd8e with 115079 events completed on d12chas552.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 134.0 MB, disk 330.0 MB, gpus: 0.0, runtime 303.04448 output: Using /cvmfs certificates. 2021-10-02 22:46:54 python_package_run(11088): setting up conda environment at /var/condor/execute/dir_8823/worker-230547-8828/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:46:54 python_package_run(11088): waiting for lock on directory /var/condor/execute/dir_8823/worker-230547-8828/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:46:55 python_package_run(11088): directory /var/condor/execute/dir_8823/worker-230547-8828/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:46:55 python_package_run(11088): activating environment 2021-10-02 22:46:55 python_package_run(11088): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_8823/worker-230547-8828/t.368/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_8823/worker-230547-8828/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_8823/worker-230547-8828/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_8823/worker-230547-8828/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_8823/worker-230547-8828/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_8823/worker-230547-8828/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_8823/worker-230547-8828/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_8823/worker-230547-8828/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_8823/worker-230547-8828/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttlnuJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_6942.root 2021-10-02 22:51:57 python_package_run(11088): command line exit code: 1 2021-10-02 22:51:57 python_package_run(11088): total runtime: 302.924 seconds task id 368 item 7434bd8e failed: OUTPUT_MISS ('UL17_ttlnuJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttlnuJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_6942.root', 'Events', 115080, 230159) 7434bd8e without result. resubmitting 7434bd8e partly as 7434bd8e. 1 attempt(s) left. processing task id 365 item 5bee00c9 with 118381 events completed on d12chas552.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 134.0 MB, disk 330.0 MB, gpus: 0.0, runtime 303.538303 output: Using /cvmfs certificates. 2021-10-02 22:46:54 python_package_run(11093): setting up conda environment at /var/condor/execute/dir_8823/worker-230547-8828/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:46:54 python_package_run(11093): waiting for lock on directory /var/condor/execute/dir_8823/worker-230547-8828/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:46:55 python_package_run(11093): directory /var/condor/execute/dir_8823/worker-230547-8828/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:46:55 python_package_run(11093): activating environment 2021-10-02 22:46:55 python_package_run(11093): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_8823/worker-230547-8828/t.365/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_8823/worker-230547-8828/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_8823/worker-230547-8828/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_8823/worker-230547-8828/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_8823/worker-230547-8828/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_8823/worker-230547-8828/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_8823/worker-230547-8828/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_8823/worker-230547-8828/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_8823/worker-230547-8828/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttlnuJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_7561.root 2021-10-02 22:51:58 python_package_run(11093): command line exit code: 1 2021-10-02 22:51:58 python_package_run(11093): total runtime: 303.420 seconds task id 365 item 5bee00c9 failed: OUTPUT_MISS ('UL17_ttlnuJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttlnuJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_7561.root', 'Events', 0, 118381) 5bee00c9 without result. resubmitting 5bee00c9 partly as 5bee00c9. 1 attempt(s) left. processing task id 366 item b9125432 with 118381 events completed on d12chas552.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 134.0 MB, disk 330.0 MB, gpus: 0.0, runtime 303.546138 output: Using /cvmfs certificates. 2021-10-02 22:46:54 python_package_run(11092): setting up conda environment at /var/condor/execute/dir_8823/worker-230547-8828/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:46:54 python_package_run(11092): waiting for lock on directory /var/condor/execute/dir_8823/worker-230547-8828/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:46:55 python_package_run(11092): directory /var/condor/execute/dir_8823/worker-230547-8828/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:46:55 python_package_run(11092): activating environment 2021-10-02 22:46:55 python_package_run(11092): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_8823/worker-230547-8828/t.366/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_8823/worker-230547-8828/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_8823/worker-230547-8828/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_8823/worker-230547-8828/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_8823/worker-230547-8828/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_8823/worker-230547-8828/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_8823/worker-230547-8828/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_8823/worker-230547-8828/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_8823/worker-230547-8828/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttlnuJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_7561.root 2021-10-02 22:51:58 python_package_run(11092): command line exit code: 1 2021-10-02 22:51:58 python_package_run(11092): total runtime: 303.427 seconds task id 366 item b9125432 failed: OUTPUT_MISS ('UL17_ttlnuJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttlnuJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_7561.root', 'Events', 118381, 236762) b9125432 without result. resubmitting b9125432 partly as b9125432. 1 attempt(s) left. processing task id 357 item 8e4ec4f4 with 94500 events completed on d12chas552.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 134.0 MB, disk 330.0 MB, gpus: 0.0, runtime 303.553365 output: Using /cvmfs certificates. 2021-10-02 22:46:54 python_package_run(11098): setting up conda environment at /var/condor/execute/dir_8823/worker-230547-8828/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:46:54 python_package_run(11098): waiting for lock on directory /var/condor/execute/dir_8823/worker-230547-8828/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:46:55 python_package_run(11098): directory /var/condor/execute/dir_8823/worker-230547-8828/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:46:55 python_package_run(11098): activating environment 2021-10-02 22:46:55 python_package_run(11098): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_8823/worker-230547-8828/t.357/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_8823/worker-230547-8828/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_8823/worker-230547-8828/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_8823/worker-230547-8828/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_8823/worker-230547-8828/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_8823/worker-230547-8828/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_8823/worker-230547-8828/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_8823/worker-230547-8828/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_8823/worker-230547-8828/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_10077.root 2021-10-02 22:51:58 python_package_run(11098): command line exit code: 1 2021-10-02 22:51:58 python_package_run(11098): total runtime: 303.439 seconds task id 357 item 8e4ec4f4 failed: OUTPUT_MISS ('UL17_tttt_b4', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_10077.root', 'Events', 0, 94500) 8e4ec4f4 without result. resubmitting 8e4ec4f4 partly as 8e4ec4f4. 1 attempt(s) left. processing task id 379 item e9636046 with 119248 events completed on d12chas548.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 132.0 MB, disk 330.0 MB, gpus: 0.0, runtime 302.811987 output: Using /cvmfs certificates. 2021-10-02 22:46:55 python_package_run(24356): setting up conda environment at /var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:46:55 python_package_run(24356): waiting for lock on directory /var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:46:55 python_package_run(24356): directory /var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:46:55 python_package_run(24356): activating environment 2021-10-02 22:46:55 python_package_run(24356): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_24697/worker-230547-24702/t.379/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttlnuJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_3625.root 2021-10-02 22:51:58 python_package_run(24356): command line exit code: 1 2021-10-02 22:51:58 python_package_run(24356): total runtime: 302.705 seconds task id 379 item e9636046 failed: OUTPUT_MISS ('UL17_ttlnuJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttlnuJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_3625.root', 'Events', 0, 119248) e9636046 without result. resubmitting e9636046 partly as e9636046. 1 attempt(s) left. processing task id 383 item 61a912bc with 138451 events completed on d12chas587.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 138.0 MB, disk 330.0 MB, gpus: 0.0, runtime 302.40855 output: Using /cvmfs certificates. 2021-10-02 22:46:56 python_package_run(31585): setting up conda environment at /var/condor/execute/dir_29058/worker-230547-29063/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:46:56 python_package_run(31585): waiting for lock on directory /var/condor/execute/dir_29058/worker-230547-29063/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:46:56 python_package_run(31585): directory /var/condor/execute/dir_29058/worker-230547-29063/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:46:56 python_package_run(31585): activating environment 2021-10-02 22:46:56 python_package_run(31585): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_29058/worker-230547-29063/t.383/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_29058/worker-230547-29063/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_29058/worker-230547-29063/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_29058/worker-230547-29063/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_29058/worker-230547-29063/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_29058/worker-230547-29063/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_29058/worker-230547-29063/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_29058/worker-230547-29063/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_29058/worker-230547-29063/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttlnuJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_17525.root 2021-10-02 22:51:58 python_package_run(31585): command line exit code: 1 2021-10-02 22:51:58 python_package_run(31585): total runtime: 302.292 seconds task id 383 item 61a912bc failed: OUTPUT_MISS ('UL17_ttlnuJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttlnuJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_17525.root', 'Events', 0, 138451) 61a912bc without result. resubmitting 61a912bc partly as 61a912bc. 1 attempt(s) left. processing task id 378 item fae745f2 with 119217 events completed on d12chas548.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 132.0 MB, disk 330.0 MB, gpus: 0.0, runtime 302.837553 output: Using /cvmfs certificates. 2021-10-02 22:46:55 python_package_run(24357): setting up conda environment at /var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:46:55 python_package_run(24357): waiting for lock on directory /var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:46:55 python_package_run(24357): directory /var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:46:55 python_package_run(24357): activating environment 2021-10-02 22:46:55 python_package_run(24357): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_24697/worker-230547-24702/t.378/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttlnuJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_4106.root 2021-10-02 22:51:58 python_package_run(24357): command line exit code: 1 2021-10-02 22:51:58 python_package_run(24357): total runtime: 302.739 seconds task id 378 item fae745f2 failed: OUTPUT_MISS ('UL17_ttlnuJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttlnuJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_4106.root', 'Events', 119218, 238435) fae745f2 without result. resubmitting fae745f2 partly as fae745f2. 1 attempt(s) left. processing task id 340 item b404b6c2 with 94250 events completed on d12chas577.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 132.0 MB, disk 330.0 MB, gpus: 0.0, runtime 302.870829 output: Using /cvmfs certificates. 2021-10-02 22:46:54 python_package_run(24962): setting up conda environment at /var/condor/execute/dir_22513/worker-230547-22528/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:46:54 python_package_run(24962): waiting for lock on directory /var/condor/execute/dir_22513/worker-230547-22528/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:46:54 python_package_run(24962): directory /var/condor/execute/dir_22513/worker-230547-22528/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:46:54 python_package_run(24962): activating environment 2021-10-02 22:46:54 python_package_run(24962): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_22513/worker-230547-22528/t.340/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_22513/worker-230547-22528/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_22513/worker-230547-22528/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_22513/worker-230547-22528/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_22513/worker-230547-22528/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_22513/worker-230547-22528/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_22513/worker-230547-22528/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_22513/worker-230547-22528/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_22513/worker-230547-22528/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_10095.root 2021-10-02 22:51:57 python_package_run(24962): command line exit code: 1 2021-10-02 22:51:57 python_package_run(24962): total runtime: 302.789 seconds task id 340 item b404b6c2 failed: OUTPUT_MISS ('UL17_tttt_b4', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_10095.root', 'Events', 94250, 188500) b404b6c2 without result. resubmitting b404b6c2 partly as b404b6c2. 1 attempt(s) left. processing task id 376 item c2ca64a7 with 119156 events completed on d12chas577.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 132.0 MB, disk 330.0 MB, gpus: 0.0, runtime 302.945612 output: Using /cvmfs certificates. 2021-10-02 22:46:55 python_package_run(25059): setting up conda environment at /var/condor/execute/dir_22513/worker-230547-22528/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:46:55 python_package_run(25059): waiting for lock on directory /var/condor/execute/dir_22513/worker-230547-22528/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:46:55 python_package_run(25059): directory /var/condor/execute/dir_22513/worker-230547-22528/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:46:55 python_package_run(25059): activating environment 2021-10-02 22:46:55 python_package_run(25059): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_22513/worker-230547-22528/t.376/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_22513/worker-230547-22528/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_22513/worker-230547-22528/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_22513/worker-230547-22528/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_22513/worker-230547-22528/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_22513/worker-230547-22528/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_22513/worker-230547-22528/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_22513/worker-230547-22528/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_22513/worker-230547-22528/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttlnuJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_4658.root 2021-10-02 22:51:58 python_package_run(25059): command line exit code: 1 2021-10-02 22:51:58 python_package_run(25059): total runtime: 302.769 seconds task id 376 item c2ca64a7 failed: OUTPUT_MISS ('UL17_ttlnuJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttlnuJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_4658.root', 'Events', 119157, 238313) c2ca64a7 without result. resubmitting c2ca64a7 partly as c2ca64a7. 1 attempt(s) left. processing task id 381 item 312973ef with 119598 events completed on d12chas577.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 133.0 MB, disk 330.0 MB, gpus: 0.0, runtime 301.476282 output: Using /cvmfs certificates. 2021-10-02 22:46:57 python_package_run(25183): setting up conda environment at /var/condor/execute/dir_22513/worker-230547-22528/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:46:57 python_package_run(25183): waiting for lock on directory /var/condor/execute/dir_22513/worker-230547-22528/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:46:57 python_package_run(25183): directory /var/condor/execute/dir_22513/worker-230547-22528/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:46:57 python_package_run(25183): activating environment 2021-10-02 22:46:57 python_package_run(25183): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_22513/worker-230547-22528/t.381/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_22513/worker-230547-22528/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_22513/worker-230547-22528/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_22513/worker-230547-22528/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_22513/worker-230547-22528/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_22513/worker-230547-22528/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_22513/worker-230547-22528/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_22513/worker-230547-22528/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_22513/worker-230547-22528/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttlnuJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_2864.root 2021-10-02 22:51:58 python_package_run(25183): command line exit code: 1 2021-10-02 22:51:58 python_package_run(25183): total runtime: 301.242 seconds task id 381 item 312973ef failed: OUTPUT_MISS ('UL17_ttlnuJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttlnuJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_2864.root', 'Events', 0, 119598) 312973ef without result. resubmitting 312973ef partly as 312973ef. 1 attempt(s) left. processing task id 385 item 14c0da6c with 108926 events completed on d12chas577.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 133.0 MB, disk 330.0 MB, gpus: 0.0, runtime 301.478071 output: Using /cvmfs certificates. 2021-10-02 22:46:57 python_package_run(25184): setting up conda environment at /var/condor/execute/dir_22513/worker-230547-22528/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:46:57 python_package_run(25184): waiting for lock on directory /var/condor/execute/dir_22513/worker-230547-22528/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:46:57 python_package_run(25184): directory /var/condor/execute/dir_22513/worker-230547-22528/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:46:57 python_package_run(25184): activating environment 2021-10-02 22:46:57 python_package_run(25184): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_22513/worker-230547-22528/t.385/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_22513/worker-230547-22528/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_22513/worker-230547-22528/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_22513/worker-230547-22528/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_22513/worker-230547-22528/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_22513/worker-230547-22528/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_22513/worker-230547-22528/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_22513/worker-230547-22528/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_22513/worker-230547-22528/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttlnuJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_16955.root 2021-10-02 22:51:58 python_package_run(25184): command line exit code: 1 2021-10-02 22:51:58 python_package_run(25184): total runtime: 301.245 seconds task id 385 item 14c0da6c failed: OUTPUT_MISS ('UL17_ttlnuJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttlnuJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_16955.root', 'Events', 108926, 217852) 14c0da6c without result. resubmitting 14c0da6c partly as 14c0da6c. 1 attempt(s) left. processing task id 387 item c9b9f8d0 with 119285 events completed on d12chas577.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 133.0 MB, disk 330.0 MB, gpus: 0.0, runtime 301.481545 output: Using /cvmfs certificates. 2021-10-02 22:46:57 python_package_run(25180): setting up conda environment at /var/condor/execute/dir_22513/worker-230547-22528/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:46:57 python_package_run(25180): waiting for lock on directory /var/condor/execute/dir_22513/worker-230547-22528/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:46:57 python_package_run(25180): directory /var/condor/execute/dir_22513/worker-230547-22528/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:46:57 python_package_run(25180): activating environment 2021-10-02 22:46:57 python_package_run(25180): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_22513/worker-230547-22528/t.387/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_22513/worker-230547-22528/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_22513/worker-230547-22528/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_22513/worker-230547-22528/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_22513/worker-230547-22528/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_22513/worker-230547-22528/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_22513/worker-230547-22528/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_22513/worker-230547-22528/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_22513/worker-230547-22528/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttlnuJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_16912.root 2021-10-02 22:51:58 python_package_run(25180): command line exit code: 1 2021-10-02 22:51:58 python_package_run(25180): total runtime: 301.339 seconds task id 387 item c9b9f8d0 failed: OUTPUT_MISS ('UL17_ttlnuJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttlnuJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_16912.root', 'Events', 119285, 238570) c9b9f8d0 without result. resubmitting c9b9f8d0 partly as c9b9f8d0. 1 attempt(s) left. processing task id 375 item fa3eb9f8 with 119157 events completed on d12chas577.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 133.0 MB, disk 330.0 MB, gpus: 0.0, runtime 303.008687 output: Using /cvmfs certificates. 2021-10-02 22:46:55 python_package_run(25060): setting up conda environment at /var/condor/execute/dir_22513/worker-230547-22528/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:46:55 python_package_run(25060): waiting for lock on directory /var/condor/execute/dir_22513/worker-230547-22528/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:46:55 python_package_run(25060): directory /var/condor/execute/dir_22513/worker-230547-22528/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:46:55 python_package_run(25060): activating environment 2021-10-02 22:46:55 python_package_run(25060): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_22513/worker-230547-22528/t.375/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_22513/worker-230547-22528/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_22513/worker-230547-22528/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_22513/worker-230547-22528/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_22513/worker-230547-22528/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_22513/worker-230547-22528/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_22513/worker-230547-22528/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_22513/worker-230547-22528/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_22513/worker-230547-22528/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttlnuJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_4658.root 2021-10-02 22:51:58 python_package_run(25060): command line exit code: 1 2021-10-02 22:51:58 python_package_run(25060): total runtime: 302.932 seconds task id 375 item fa3eb9f8 failed: OUTPUT_MISS ('UL17_ttlnuJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttlnuJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_4658.root', 'Events', 0, 119157) fa3eb9f8 without result. resubmitting fa3eb9f8 partly as fa3eb9f8. 1 attempt(s) left. processing task id 386 item 8d469b3d with 119285 events completed on d12chas577.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 133.0 MB, disk 330.0 MB, gpus: 0.0, runtime 301.551406 output: Using /cvmfs certificates. 2021-10-02 22:46:57 python_package_run(25182): setting up conda environment at /var/condor/execute/dir_22513/worker-230547-22528/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:46:57 python_package_run(25182): waiting for lock on directory /var/condor/execute/dir_22513/worker-230547-22528/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:46:57 python_package_run(25182): directory /var/condor/execute/dir_22513/worker-230547-22528/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:46:57 python_package_run(25182): activating environment 2021-10-02 22:46:57 python_package_run(25182): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_22513/worker-230547-22528/t.386/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_22513/worker-230547-22528/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_22513/worker-230547-22528/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_22513/worker-230547-22528/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_22513/worker-230547-22528/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_22513/worker-230547-22528/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_22513/worker-230547-22528/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_22513/worker-230547-22528/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_22513/worker-230547-22528/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttlnuJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_16912.root 2021-10-02 22:51:58 python_package_run(25182): command line exit code: 1 2021-10-02 22:51:58 python_package_run(25182): total runtime: 301.484 seconds task id 386 item 8d469b3d failed: OUTPUT_MISS ('UL17_ttlnuJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttlnuJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_16912.root', 'Events', 0, 119285) 8d469b3d without result. resubmitting 8d469b3d partly as 8d469b3d. 1 attempt(s) left. processing task id 377 item 57830eba with 119218 events completed on d12chas548.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 132.0 MB, disk 330.0 MB, gpus: 0.0, runtime 303.24602 output: Using /cvmfs certificates. 2021-10-02 22:46:55 python_package_run(24358): setting up conda environment at /var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:46:55 python_package_run(24358): waiting for lock on directory /var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:46:55 python_package_run(24358): directory /var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:46:55 python_package_run(24358): activating environment 2021-10-02 22:46:55 python_package_run(24358): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_24697/worker-230547-24702/t.377/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttlnuJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_4106.root 2021-10-02 22:51:58 python_package_run(24358): command line exit code: 1 2021-10-02 22:51:58 python_package_run(24358): total runtime: 303.138 seconds task id 377 item 57830eba failed: OUTPUT_MISS ('UL17_ttlnuJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttlnuJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_4106.root', 'Events', 0, 119218) 57830eba without result. resubmitting 57830eba partly as 57830eba. 1 attempt(s) left. processing task id 382 item e2a5c979 with 119598 events completed on d12chas587.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 134.0 MB, disk 330.0 MB, gpus: 0.0, runtime 302.841832 output: Using /cvmfs certificates. 2021-10-02 22:46:56 python_package_run(31586): setting up conda environment at /var/condor/execute/dir_29058/worker-230547-29063/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:46:56 python_package_run(31586): waiting for lock on directory /var/condor/execute/dir_29058/worker-230547-29063/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:46:56 python_package_run(31586): directory /var/condor/execute/dir_29058/worker-230547-29063/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:46:56 python_package_run(31586): activating environment 2021-10-02 22:46:56 python_package_run(31586): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_29058/worker-230547-29063/t.382/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_29058/worker-230547-29063/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_29058/worker-230547-29063/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_29058/worker-230547-29063/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_29058/worker-230547-29063/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_29058/worker-230547-29063/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_29058/worker-230547-29063/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_29058/worker-230547-29063/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_29058/worker-230547-29063/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttlnuJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_2864.root 2021-10-02 22:51:58 python_package_run(31586): command line exit code: 1 2021-10-02 22:51:58 python_package_run(31586): total runtime: 302.731 seconds task id 382 item e2a5c979 failed: OUTPUT_MISS ('UL17_ttlnuJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttlnuJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_2864.root', 'Events', 119598, 239196) e2a5c979 without result. resubmitting e2a5c979 partly as e2a5c979. 1 attempt(s) left. processing task id 384 item 983e90d7 with 108926 events completed on d12chas587.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 134.0 MB, disk 330.0 MB, gpus: 0.0, runtime 303.217117 output: Using /cvmfs certificates. 2021-10-02 22:46:56 python_package_run(31584): setting up conda environment at /var/condor/execute/dir_29058/worker-230547-29063/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:46:56 python_package_run(31584): waiting for lock on directory /var/condor/execute/dir_29058/worker-230547-29063/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:46:56 python_package_run(31584): directory /var/condor/execute/dir_29058/worker-230547-29063/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:46:56 python_package_run(31584): activating environment 2021-10-02 22:46:56 python_package_run(31584): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_29058/worker-230547-29063/t.384/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_29058/worker-230547-29063/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_29058/worker-230547-29063/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_29058/worker-230547-29063/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_29058/worker-230547-29063/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_29058/worker-230547-29063/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_29058/worker-230547-29063/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_29058/worker-230547-29063/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_29058/worker-230547-29063/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttlnuJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_16955.root 2021-10-02 22:51:59 python_package_run(31584): command line exit code: 1 2021-10-02 22:51:59 python_package_run(31584): total runtime: 303.102 seconds task id 384 item 983e90d7 failed: OUTPUT_MISS ('UL17_ttlnuJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttlnuJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_16955.root', 'Events', 0, 108926) 983e90d7 without result. resubmitting 983e90d7 partly as 983e90d7. 1 attempt(s) left. processing task id 391 item fd8b6091 with 112238 events completed on d12chas587.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 134.0 MB, disk 330.0 MB, gpus: 0.0, runtime 302.98908 output: Using /cvmfs certificates. 2021-10-02 22:46:57 python_package_run(31738): setting up conda environment at /var/condor/execute/dir_29058/worker-230547-29063/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:46:57 python_package_run(31738): waiting for lock on directory /var/condor/execute/dir_29058/worker-230547-29063/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:46:57 python_package_run(31738): directory /var/condor/execute/dir_29058/worker-230547-29063/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:46:57 python_package_run(31738): activating environment 2021-10-02 22:46:57 python_package_run(31738): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_29058/worker-230547-29063/t.391/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_29058/worker-230547-29063/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_29058/worker-230547-29063/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_29058/worker-230547-29063/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_29058/worker-230547-29063/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_29058/worker-230547-29063/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_29058/worker-230547-29063/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_29058/worker-230547-29063/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_29058/worker-230547-29063/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttlnuJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_16126.root 2021-10-02 22:52:00 python_package_run(31738): command line exit code: 1 2021-10-02 22:52:00 python_package_run(31738): total runtime: 302.868 seconds task id 391 item fd8b6091 failed: OUTPUT_MISS ('UL17_ttlnuJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttlnuJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_16126.root', 'Events', 112239, 224477) fd8b6091 without result. resubmitting fd8b6091 partly as fd8b6091. 1 attempt(s) left. processing task id 392 item d8131585 with 117530 events completed on d12chas552.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 134.0 MB, disk 330.0 MB, gpus: 0.0, runtime 303.240161 output: Using /cvmfs certificates. 2021-10-02 22:46:59 python_package_run(11633): setting up conda environment at /var/condor/execute/dir_8823/worker-230547-8828/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:00 python_package_run(11633): waiting for lock on directory /var/condor/execute/dir_8823/worker-230547-8828/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:00 python_package_run(11633): directory /var/condor/execute/dir_8823/worker-230547-8828/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:47:00 python_package_run(11633): activating environment 2021-10-02 22:47:00 python_package_run(11633): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_8823/worker-230547-8828/t.392/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_8823/worker-230547-8828/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_8823/worker-230547-8828/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_8823/worker-230547-8828/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_8823/worker-230547-8828/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_8823/worker-230547-8828/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_8823/worker-230547-8828/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_8823/worker-230547-8828/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_8823/worker-230547-8828/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttlnuJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_15379.root 2021-10-02 22:52:03 python_package_run(11633): command line exit code: 1 2021-10-02 22:52:03 python_package_run(11633): total runtime: 303.104 seconds task id 392 item d8131585 failed: OUTPUT_MISS ('UL17_ttlnuJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttlnuJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_15379.root', 'Events', 0, 117530) d8131585 without result. resubmitting d8131585 partly as d8131585. 1 attempt(s) left. processing task id 393 item 97ee91e3 with 117530 events completed on d12chas548.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 132.0 MB, disk 330.0 MB, gpus: 0.0, runtime 303.053567 output: Using /cvmfs certificates. 2021-10-02 22:47:02 python_package_run(24706): setting up conda environment at /var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:02 python_package_run(24706): waiting for lock on directory /var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:02 python_package_run(24706): directory /var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:47:02 python_package_run(24706): activating environment 2021-10-02 22:47:02 python_package_run(24706): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_24697/worker-230547-24702/t.393/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttlnuJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_15379.root 2021-10-02 22:52:05 python_package_run(24706): command line exit code: 1 2021-10-02 22:52:05 python_package_run(24706): total runtime: 302.935 seconds task id 393 item 97ee91e3 failed: OUTPUT_MISS ('UL17_ttlnuJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttlnuJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_15379.root', 'Events', 117530, 235060) 97ee91e3 without result. resubmitting 97ee91e3 partly as 97ee91e3. 1 attempt(s) left. processing task id 358 item 6cf8a98a with 94500 events completed on d12chas552.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 134.0 MB, disk 330.0 MB, gpus: 0.0, runtime 312.38122 output: Using /cvmfs certificates. 2021-10-02 22:46:54 python_package_run(11094): setting up conda environment at /var/condor/execute/dir_8823/worker-230547-8828/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:46:54 python_package_run(11094): waiting for lock on directory /var/condor/execute/dir_8823/worker-230547-8828/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:46:55 python_package_run(11094): directory /var/condor/execute/dir_8823/worker-230547-8828/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:46:55 python_package_run(11094): activating environment 2021-10-02 22:46:55 python_package_run(11094): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_8823/worker-230547-8828/t.358/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_8823/worker-230547-8828/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_8823/worker-230547-8828/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_8823/worker-230547-8828/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_8823/worker-230547-8828/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_8823/worker-230547-8828/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_8823/worker-230547-8828/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_8823/worker-230547-8828/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_8823/worker-230547-8828/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_10077.root 2021-10-02 22:52:07 python_package_run(11094): command line exit code: 1 2021-10-02 22:52:07 python_package_run(11094): total runtime: 312.262 seconds task id 358 item 6cf8a98a failed: OUTPUT_MISS ('UL17_tttt_b4', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_10077.root', 'Events', 94500, 189000) 6cf8a98a without result. resubmitting 6cf8a98a partly as 6cf8a98a. 1 attempt(s) left. processing task id 388 item fdbf65b4 with 116057 events completed on d12chas584.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.009, memory: 132.0 MB, disk 330.0 MB, gpus: 0.0, runtime 302.749951 output: Using /cvmfs certificates. 2021-10-02 22:47:05 python_package_run(18009): setting up conda environment at /var/condor/execute/dir_15741/worker-230547-15747/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:05 python_package_run(18009): waiting for lock on directory /var/condor/execute/dir_15741/worker-230547-15747/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:05 python_package_run(18009): directory /var/condor/execute/dir_15741/worker-230547-15747/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:47:05 python_package_run(18009): activating environment 2021-10-02 22:47:05 python_package_run(18009): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_15741/worker-230547-15747/t.388/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_15741/worker-230547-15747/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_15741/worker-230547-15747/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_15741/worker-230547-15747/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_15741/worker-230547-15747/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_15741/worker-230547-15747/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_15741/worker-230547-15747/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_15741/worker-230547-15747/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_15741/worker-230547-15747/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttlnuJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_16898.root 2021-10-02 22:52:08 python_package_run(18009): command line exit code: 1 2021-10-02 22:52:08 python_package_run(18009): total runtime: 302.635 seconds task id 388 item fdbf65b4 failed: OUTPUT_MISS ('UL17_ttlnuJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttlnuJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_16898.root', 'Events', 0, 116057) fdbf65b4 without result. resubmitting fdbf65b4 partly as fdbf65b4. 1 attempt(s) left. processing task id 370 item b471bce2 with 112549 events completed on d12chas589.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 139.0 MB, disk 330.0 MB, gpus: 0.0, runtime 302.827178 output: Using /cvmfs certificates. 2021-10-02 22:47:05 python_package_run(19797): setting up conda environment at /var/condor/execute/dir_17507/worker-230547-17512/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:05 python_package_run(19797): waiting for lock on directory /var/condor/execute/dir_17507/worker-230547-17512/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:05 python_package_run(19797): directory /var/condor/execute/dir_17507/worker-230547-17512/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:47:05 python_package_run(19797): activating environment 2021-10-02 22:47:05 python_package_run(19797): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_17507/worker-230547-17512/t.370/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_17507/worker-230547-17512/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_17507/worker-230547-17512/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_17507/worker-230547-17512/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_17507/worker-230547-17512/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_17507/worker-230547-17512/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_17507/worker-230547-17512/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_17507/worker-230547-17512/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_17507/worker-230547-17512/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttlnuJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_6353.root 2021-10-02 22:52:08 python_package_run(19797): command line exit code: 1 2021-10-02 22:52:08 python_package_run(19797): total runtime: 302.711 seconds task id 370 item b471bce2 failed: OUTPUT_MISS ('UL17_ttlnuJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttlnuJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_6353.root', 'Events', 112549, 225098) b471bce2 without result. resubmitting b471bce2 partly as b471bce2. 1 attempt(s) left. processing task id 372 item c712bfc5 with 119522 events completed on d12chas589.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 140.0 MB, disk 330.0 MB, gpus: 0.0, runtime 302.945343 output: Using /cvmfs certificates. 2021-10-02 22:47:05 python_package_run(19795): setting up conda environment at /var/condor/execute/dir_17507/worker-230547-17512/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:05 python_package_run(19795): waiting for lock on directory /var/condor/execute/dir_17507/worker-230547-17512/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:05 python_package_run(19795): directory /var/condor/execute/dir_17507/worker-230547-17512/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:47:05 python_package_run(19795): activating environment 2021-10-02 22:47:05 python_package_run(19795): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_17507/worker-230547-17512/t.372/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_17507/worker-230547-17512/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_17507/worker-230547-17512/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_17507/worker-230547-17512/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_17507/worker-230547-17512/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_17507/worker-230547-17512/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_17507/worker-230547-17512/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_17507/worker-230547-17512/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_17507/worker-230547-17512/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttlnuJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_5734.root 2021-10-02 22:52:08 python_package_run(19795): command line exit code: 1 2021-10-02 22:52:08 python_package_run(19795): total runtime: 302.825 seconds task id 372 item c712bfc5 failed: OUTPUT_MISS ('UL17_ttlnuJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttlnuJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_5734.root', 'Events', 119522, 239044) c712bfc5 without result. resubmitting c712bfc5 partly as c712bfc5. 1 attempt(s) left. processing task id 374 item d84b939c with 119232 events completed on d12chas589.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 134.0 MB, disk 330.0 MB, gpus: 0.0, runtime 302.949909 output: Using /cvmfs certificates. 2021-10-02 22:47:05 python_package_run(19793): setting up conda environment at /var/condor/execute/dir_17507/worker-230547-17512/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:05 python_package_run(19793): waiting for lock on directory /var/condor/execute/dir_17507/worker-230547-17512/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:05 python_package_run(19793): directory /var/condor/execute/dir_17507/worker-230547-17512/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:47:05 python_package_run(19793): activating environment 2021-10-02 22:47:05 python_package_run(19793): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_17507/worker-230547-17512/t.374/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_17507/worker-230547-17512/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_17507/worker-230547-17512/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_17507/worker-230547-17512/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_17507/worker-230547-17512/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_17507/worker-230547-17512/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_17507/worker-230547-17512/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_17507/worker-230547-17512/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_17507/worker-230547-17512/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttlnuJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_5336.root 2021-10-02 22:52:08 python_package_run(19793): command line exit code: 1 2021-10-02 22:52:08 python_package_run(19793): total runtime: 302.834 seconds task id 374 item d84b939c failed: OUTPUT_MISS ('UL17_ttlnuJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttlnuJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_5336.root', 'Events', 119233, 238465) d84b939c without result. resubmitting d84b939c partly as d84b939c. 1 attempt(s) left. processing task id 389 item 5b67d2af with 116056 events completed on d12chas584.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 134.0 MB, disk 330.0 MB, gpus: 0.0, runtime 303.136361 output: Using /cvmfs certificates. 2021-10-02 22:47:05 python_package_run(18008): setting up conda environment at /var/condor/execute/dir_15741/worker-230547-15747/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:05 python_package_run(18008): waiting for lock on directory /var/condor/execute/dir_15741/worker-230547-15747/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:05 python_package_run(18008): directory /var/condor/execute/dir_15741/worker-230547-15747/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:47:05 python_package_run(18008): activating environment 2021-10-02 22:47:05 python_package_run(18008): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_15741/worker-230547-15747/t.389/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_15741/worker-230547-15747/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_15741/worker-230547-15747/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_15741/worker-230547-15747/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_15741/worker-230547-15747/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_15741/worker-230547-15747/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_15741/worker-230547-15747/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_15741/worker-230547-15747/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_15741/worker-230547-15747/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttlnuJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_16898.root 2021-10-02 22:52:08 python_package_run(18008): command line exit code: 1 2021-10-02 22:52:08 python_package_run(18008): total runtime: 303.020 seconds task id 389 item 5b67d2af failed: OUTPUT_MISS ('UL17_ttlnuJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttlnuJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_16898.root', 'Events', 116057, 232113) 5b67d2af without result. resubmitting 5b67d2af partly as 5b67d2af. 1 attempt(s) left. processing task id 380 item 9d220c0e with 119248 events completed on d12chas589.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 140.0 MB, disk 330.0 MB, gpus: 0.0, runtime 303.256163 output: Using /cvmfs certificates. 2021-10-02 22:47:05 python_package_run(19791): setting up conda environment at /var/condor/execute/dir_17507/worker-230547-17512/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:05 python_package_run(19791): waiting for lock on directory /var/condor/execute/dir_17507/worker-230547-17512/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:05 python_package_run(19791): directory /var/condor/execute/dir_17507/worker-230547-17512/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:47:05 python_package_run(19791): activating environment 2021-10-02 22:47:05 python_package_run(19791): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_17507/worker-230547-17512/t.380/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_17507/worker-230547-17512/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_17507/worker-230547-17512/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_17507/worker-230547-17512/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_17507/worker-230547-17512/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_17507/worker-230547-17512/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_17507/worker-230547-17512/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_17507/worker-230547-17512/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_17507/worker-230547-17512/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttlnuJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_3625.root 2021-10-02 22:52:08 python_package_run(19791): command line exit code: 1 2021-10-02 22:52:08 python_package_run(19791): total runtime: 303.135 seconds task id 380 item 9d220c0e failed: OUTPUT_MISS ('UL17_ttlnuJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttlnuJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_3625.root', 'Events', 119248, 238496) 9d220c0e without result. resubmitting 9d220c0e partly as 9d220c0e. 1 attempt(s) left. processing task id 373 item 42b3e2b8 with 119233 events completed on d12chas589.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 140.0 MB, disk 330.0 MB, gpus: 0.0, runtime 303.285904 output: Using /cvmfs certificates. 2021-10-02 22:47:05 python_package_run(19794): setting up conda environment at /var/condor/execute/dir_17507/worker-230547-17512/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:05 python_package_run(19794): waiting for lock on directory /var/condor/execute/dir_17507/worker-230547-17512/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:05 python_package_run(19794): directory /var/condor/execute/dir_17507/worker-230547-17512/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:47:05 python_package_run(19794): activating environment 2021-10-02 22:47:06 python_package_run(19794): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_17507/worker-230547-17512/t.373/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_17507/worker-230547-17512/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_17507/worker-230547-17512/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_17507/worker-230547-17512/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_17507/worker-230547-17512/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_17507/worker-230547-17512/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_17507/worker-230547-17512/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_17507/worker-230547-17512/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_17507/worker-230547-17512/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttlnuJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_5336.root 2021-10-02 22:52:08 python_package_run(19794): command line exit code: 1 2021-10-02 22:52:08 python_package_run(19794): total runtime: 303.170 seconds task id 373 item 42b3e2b8 failed: OUTPUT_MISS ('UL17_ttlnuJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttlnuJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_5336.root', 'Events', 0, 119233) 42b3e2b8 without result. resubmitting 42b3e2b8 partly as 42b3e2b8. 1 attempt(s) left. processing task id 371 item ba606236 with 119522 events completed on d12chas589.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 134.0 MB, disk 330.0 MB, gpus: 0.0, runtime 303.305499 output: Using /cvmfs certificates. 2021-10-02 22:47:05 python_package_run(19796): setting up conda environment at /var/condor/execute/dir_17507/worker-230547-17512/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:05 python_package_run(19796): waiting for lock on directory /var/condor/execute/dir_17507/worker-230547-17512/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:06 python_package_run(19796): directory /var/condor/execute/dir_17507/worker-230547-17512/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:47:06 python_package_run(19796): activating environment 2021-10-02 22:47:06 python_package_run(19796): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_17507/worker-230547-17512/t.371/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_17507/worker-230547-17512/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_17507/worker-230547-17512/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_17507/worker-230547-17512/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_17507/worker-230547-17512/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_17507/worker-230547-17512/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_17507/worker-230547-17512/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_17507/worker-230547-17512/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_17507/worker-230547-17512/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttlnuJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_5734.root 2021-10-02 22:52:08 python_package_run(19796): command line exit code: 1 2021-10-02 22:52:08 python_package_run(19796): total runtime: 303.188 seconds task id 371 item ba606236 failed: OUTPUT_MISS ('UL17_ttlnuJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttlnuJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_5734.root', 'Events', 0, 119522) ba606236 without result. resubmitting ba606236 partly as ba606236. 1 attempt(s) left. processing task id 394 item 76a6d86f with 117615 events completed on d12chas584.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 132.0 MB, disk 330.0 MB, gpus: 0.0, runtime 303.554387 output: Using /cvmfs certificates. 2021-10-02 22:47:05 python_package_run(18006): setting up conda environment at /var/condor/execute/dir_15741/worker-230547-15747/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:05 python_package_run(18006): waiting for lock on directory /var/condor/execute/dir_15741/worker-230547-15747/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:05 python_package_run(18006): directory /var/condor/execute/dir_15741/worker-230547-15747/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:47:05 python_package_run(18006): activating environment 2021-10-02 22:47:05 python_package_run(18006): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_15741/worker-230547-15747/t.394/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_15741/worker-230547-15747/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_15741/worker-230547-15747/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_15741/worker-230547-15747/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_15741/worker-230547-15747/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_15741/worker-230547-15747/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_15741/worker-230547-15747/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_15741/worker-230547-15747/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_15741/worker-230547-15747/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttlnuJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_14544.root 2021-10-02 22:52:08 python_package_run(18006): command line exit code: 1 2021-10-02 22:52:09 python_package_run(18006): total runtime: 303.445 seconds task id 394 item 76a6d86f failed: OUTPUT_MISS ('UL17_ttlnuJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttlnuJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_14544.root', 'Events', 0, 117615) 76a6d86f without result. resubmitting 76a6d86f partly as 76a6d86f. 1 attempt(s) left. processing task id 390 item 566bf5de with 112239 events completed on d12chas584.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 133.0 MB, disk 330.0 MB, gpus: 0.0, runtime 303.611483 output: Using /cvmfs certificates. 2021-10-02 22:47:05 python_package_run(18007): setting up conda environment at /var/condor/execute/dir_15741/worker-230547-15747/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:05 python_package_run(18007): waiting for lock on directory /var/condor/execute/dir_15741/worker-230547-15747/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:05 python_package_run(18007): directory /var/condor/execute/dir_15741/worker-230547-15747/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:47:05 python_package_run(18007): activating environment 2021-10-02 22:47:05 python_package_run(18007): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_15741/worker-230547-15747/t.390/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_15741/worker-230547-15747/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_15741/worker-230547-15747/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_15741/worker-230547-15747/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_15741/worker-230547-15747/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_15741/worker-230547-15747/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_15741/worker-230547-15747/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_15741/worker-230547-15747/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_15741/worker-230547-15747/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttlnuJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_16126.root 2021-10-02 22:52:09 python_package_run(18007): command line exit code: 1 2021-10-02 22:52:09 python_package_run(18007): total runtime: 303.497 seconds task id 390 item 566bf5de failed: OUTPUT_MISS ('UL17_ttlnuJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttlnuJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_16126.root', 'Events', 0, 112239) 566bf5de without result. resubmitting 566bf5de partly as 566bf5de. 1 attempt(s) left. processing task id 404 item 96d6a8ff with 119035 events completed on d12chas589.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.009, memory: 138.0 MB, disk 330.0 MB, gpus: 0.0, runtime 302.756878 output: Using /cvmfs certificates. 2021-10-02 22:47:07 python_package_run(20073): setting up conda environment at /var/condor/execute/dir_17507/worker-230547-17512/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:07 python_package_run(20073): waiting for lock on directory /var/condor/execute/dir_17507/worker-230547-17512/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:07 python_package_run(20073): directory /var/condor/execute/dir_17507/worker-230547-17512/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:47:07 python_package_run(20073): activating environment 2021-10-02 22:47:07 python_package_run(20073): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_17507/worker-230547-17512/t.404/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_17507/worker-230547-17512/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_17507/worker-230547-17512/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_17507/worker-230547-17512/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_17507/worker-230547-17512/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_17507/worker-230547-17512/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_17507/worker-230547-17512/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_17507/worker-230547-17512/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_17507/worker-230547-17512/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttlnuJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_12120.root 2021-10-02 22:52:09 python_package_run(20073): command line exit code: 1 2021-10-02 22:52:09 python_package_run(20073): total runtime: 302.642 seconds task id 404 item 96d6a8ff failed: OUTPUT_MISS ('UL17_ttlnuJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttlnuJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_12120.root', 'Events', 119035, 238070) 96d6a8ff without result. resubmitting 96d6a8ff partly as 96d6a8ff. 1 attempt(s) left. processing task id 403 item c52427a6 with 119035 events completed on d12chas577.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 133.0 MB, disk 330.0 MB, gpus: 0.0, runtime 302.771319 output: Using /cvmfs certificates. 2021-10-02 22:47:07 python_package_run(25428): setting up conda environment at /var/condor/execute/dir_22513/worker-230547-22528/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:07 python_package_run(25428): waiting for lock on directory /var/condor/execute/dir_22513/worker-230547-22528/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:07 python_package_run(25428): directory /var/condor/execute/dir_22513/worker-230547-22528/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:47:07 python_package_run(25428): activating environment 2021-10-02 22:47:07 python_package_run(25428): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_22513/worker-230547-22528/t.403/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_22513/worker-230547-22528/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_22513/worker-230547-22528/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_22513/worker-230547-22528/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_22513/worker-230547-22528/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_22513/worker-230547-22528/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_22513/worker-230547-22528/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_22513/worker-230547-22528/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_22513/worker-230547-22528/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttlnuJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_12120.root 2021-10-02 22:52:09 python_package_run(25428): command line exit code: 1 2021-10-02 22:52:09 python_package_run(25428): total runtime: 302.664 seconds task id 403 item c52427a6 failed: OUTPUT_MISS ('UL17_ttlnuJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttlnuJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_12120.root', 'Events', 0, 119035) c52427a6 without result. resubmitting c52427a6 partly as c52427a6. 1 attempt(s) left. processing task id 406 item 9006fc01 with 112931 events completed on qa-rtx6k-021.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.007, memory: 132.0 MB, disk 330.0 MB, gpus: 0.0, runtime 302.392938 output: Using /cvmfs certificates. 2021-10-02 22:47:08 python_package_run(159458): setting up conda environment at /var/condor/execute/dir_155375/worker-230547-155384/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:08 python_package_run(159458): waiting for lock on directory /var/condor/execute/dir_155375/worker-230547-155384/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:08 python_package_run(159458): directory /var/condor/execute/dir_155375/worker-230547-155384/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:47:08 python_package_run(159458): activating environment 2021-10-02 22:47:08 python_package_run(159458): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_155375/worker-230547-155384/t.406/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_155375/worker-230547-155384/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_155375/worker-230547-155384/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_155375/worker-230547-155384/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_155375/worker-230547-155384/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_155375/worker-230547-155384/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_155375/worker-230547-155384/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_155375/worker-230547-155384/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_155375/worker-230547-155384/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttlnuJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_11500.root 2021-10-02 22:52:10 python_package_run(159458): command line exit code: 1 2021-10-02 22:52:10 python_package_run(159458): total runtime: 302.302 seconds task id 406 item 9006fc01 failed: OUTPUT_MISS ('UL17_ttlnuJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttlnuJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_11500.root', 'Events', 112931, 225862) 9006fc01 without result. resubmitting 9006fc01 partly as 9006fc01. 1 attempt(s) left. processing task id 407 item 1369a390 with 112521 events completed on qa-rtx6k-021.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.007, memory: 132.0 MB, disk 330.0 MB, gpus: 0.0, runtime 302.427089 output: Using /cvmfs certificates. 2021-10-02 22:47:08 python_package_run(159455): setting up conda environment at /var/condor/execute/dir_155375/worker-230547-155384/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:08 python_package_run(159455): waiting for lock on directory /var/condor/execute/dir_155375/worker-230547-155384/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:08 python_package_run(159455): directory /var/condor/execute/dir_155375/worker-230547-155384/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:47:08 python_package_run(159455): activating environment 2021-10-02 22:47:08 python_package_run(159455): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_155375/worker-230547-155384/t.407/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_155375/worker-230547-155384/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_155375/worker-230547-155384/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_155375/worker-230547-155384/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_155375/worker-230547-155384/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_155375/worker-230547-155384/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_155375/worker-230547-155384/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_155375/worker-230547-155384/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_155375/worker-230547-155384/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttlnuJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_10889.root 2021-10-02 22:52:10 python_package_run(159455): command line exit code: 1 2021-10-02 22:52:10 python_package_run(159455): total runtime: 302.329 seconds task id 407 item 1369a390 failed: OUTPUT_MISS ('UL17_ttlnuJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttlnuJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_10889.root', 'Events', 0, 112521) 1369a390 without result. resubmitting 1369a390 partly as 1369a390. 1 attempt(s) left. processing task id 408 item 2ed8cb5b with 112521 events completed on qa-rtx6k-021.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.007, memory: 132.0 MB, disk 330.0 MB, gpus: 0.0, runtime 302.461834 output: Using /cvmfs certificates. 2021-10-02 22:47:08 python_package_run(159453): setting up conda environment at /var/condor/execute/dir_155375/worker-230547-155384/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:08 python_package_run(159453): waiting for lock on directory /var/condor/execute/dir_155375/worker-230547-155384/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:08 python_package_run(159453): directory /var/condor/execute/dir_155375/worker-230547-155384/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:47:08 python_package_run(159453): activating environment 2021-10-02 22:47:08 python_package_run(159453): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_155375/worker-230547-155384/t.408/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_155375/worker-230547-155384/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_155375/worker-230547-155384/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_155375/worker-230547-155384/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_155375/worker-230547-155384/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_155375/worker-230547-155384/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_155375/worker-230547-155384/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_155375/worker-230547-155384/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_155375/worker-230547-155384/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttlnuJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_10889.root 2021-10-02 22:52:10 python_package_run(159453): command line exit code: 1 2021-10-02 22:52:10 python_package_run(159453): total runtime: 302.374 seconds task id 408 item 2ed8cb5b failed: OUTPUT_MISS ('UL17_ttlnuJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttlnuJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_10889.root', 'Events', 112521, 225042) 2ed8cb5b without result. resubmitting 2ed8cb5b partly as 2ed8cb5b. 1 attempt(s) left. processing task id 398 item 8b2a2c8c with 119578 events completed on d12chas584.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 132.0 MB, disk 330.0 MB, gpus: 0.0, runtime 301.897892 output: Using /cvmfs certificates. 2021-10-02 22:47:10 python_package_run(18243): setting up conda environment at /var/condor/execute/dir_15741/worker-230547-15747/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:10 python_package_run(18243): waiting for lock on directory /var/condor/execute/dir_15741/worker-230547-15747/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:10 python_package_run(18243): directory /var/condor/execute/dir_15741/worker-230547-15747/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:47:10 python_package_run(18243): activating environment 2021-10-02 22:47:10 python_package_run(18243): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_15741/worker-230547-15747/t.398/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_15741/worker-230547-15747/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_15741/worker-230547-15747/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_15741/worker-230547-15747/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_15741/worker-230547-15747/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_15741/worker-230547-15747/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_15741/worker-230547-15747/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_15741/worker-230547-15747/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_15741/worker-230547-15747/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttlnuJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_13961.root 2021-10-02 22:52:12 python_package_run(18243): command line exit code: 1 2021-10-02 22:52:12 python_package_run(18243): total runtime: 301.783 seconds task id 398 item 8b2a2c8c failed: OUTPUT_MISS ('UL17_ttlnuJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttlnuJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_13961.root', 'Events', 119579, 239157) 8b2a2c8c without result. resubmitting 8b2a2c8c partly as 8b2a2c8c. 1 attempt(s) left. processing task id 451 item 5e3d8b77 with 123773 events completed on d12chas545.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 134.0 MB, disk 330.0 MB, gpus: 0.0, runtime 293.150952 output: Using /cvmfs certificates. 2021-10-02 22:47:19 python_package_run(28340): setting up conda environment at /var/condor/execute/dir_25722/worker-230547-25727/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:19 python_package_run(28340): waiting for lock on directory /var/condor/execute/dir_25722/worker-230547-25727/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:19 python_package_run(28340): directory /var/condor/execute/dir_25722/worker-230547-25727/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:47:19 python_package_run(28340): activating environment 2021-10-02 22:47:19 python_package_run(28340): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_25722/worker-230547-25727/t.451/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_25722/worker-230547-25727/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_25722/worker-230547-25727/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_25722/worker-230547-25727/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_25722/worker-230547-25727/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_25722/worker-230547-25727/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_25722/worker-230547-25727/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_25722/worker-230547-25727/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_25722/worker-230547-25727/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttllNuNuJetNoHiggs_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_14543.root 2021-10-02 22:52:12 python_package_run(28340): command line exit code: 1 2021-10-02 22:52:12 python_package_run(28340): total runtime: 293.021 seconds task id 451 item 5e3d8b77 failed: OUTPUT_MISS ('UL17_ttllJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttllNuNuJetNoHiggs_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_14543.root', 'Events', 123774, 247547) 5e3d8b77 without result. resubmitting 5e3d8b77 partly as 5e3d8b77. 1 attempt(s) left. processing task id 449 item 6d20edce with 124612 events completed on d12chas564.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 132.0 MB, disk 330.0 MB, gpus: 0.0, runtime 293.180206 output: Using /cvmfs certificates. 2021-10-02 22:47:19 python_package_run(25974): setting up conda environment at /var/condor/execute/dir_23236/worker-230547-23258/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:19 python_package_run(25974): waiting for lock on directory /var/condor/execute/dir_23236/worker-230547-23258/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:19 python_package_run(25974): directory /var/condor/execute/dir_23236/worker-230547-23258/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:47:19 python_package_run(25974): activating environment 2021-10-02 22:47:19 python_package_run(25974): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_23236/worker-230547-23258/t.449/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_23236/worker-230547-23258/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_23236/worker-230547-23258/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_23236/worker-230547-23258/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_23236/worker-230547-23258/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_23236/worker-230547-23258/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_23236/worker-230547-23258/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_23236/worker-230547-23258/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_23236/worker-230547-23258/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttllNuNuJetNoHiggs_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_15782.root 2021-10-02 22:52:12 python_package_run(25974): command line exit code: 1 2021-10-02 22:52:12 python_package_run(25974): total runtime: 293.080 seconds task id 449 item 6d20edce failed: OUTPUT_MISS ('UL17_ttllJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttllNuNuJetNoHiggs_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_15782.root', 'Events', 124612, 249224) 6d20edce without result. resubmitting 6d20edce partly as 6d20edce. 1 attempt(s) left. processing task id 450 item 4dbb2b4e with 123774 events completed on d12chas551.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.009, memory: 132.0 MB, disk 330.0 MB, gpus: 0.0, runtime 293.189807 output: Using /cvmfs certificates. 2021-10-02 22:47:19 python_package_run(9643): setting up conda environment at /var/condor/execute/dir_7659/worker-230547-7664/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:19 python_package_run(9643): waiting for lock on directory /var/condor/execute/dir_7659/worker-230547-7664/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:19 python_package_run(9643): directory /var/condor/execute/dir_7659/worker-230547-7664/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:47:19 python_package_run(9643): activating environment 2021-10-02 22:47:19 python_package_run(9643): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_7659/worker-230547-7664/t.450/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_7659/worker-230547-7664/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_7659/worker-230547-7664/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_7659/worker-230547-7664/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_7659/worker-230547-7664/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_7659/worker-230547-7664/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_7659/worker-230547-7664/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_7659/worker-230547-7664/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_7659/worker-230547-7664/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttllNuNuJetNoHiggs_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_14543.root 2021-10-02 22:52:12 python_package_run(9643): command line exit code: 1 2021-10-02 22:52:12 python_package_run(9643): total runtime: 293.097 seconds task id 450 item 4dbb2b4e failed: OUTPUT_MISS ('UL17_ttllJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttllNuNuJetNoHiggs_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_14543.root', 'Events', 0, 123774) 4dbb2b4e without result. resubmitting 4dbb2b4e partly as 4dbb2b4e. 1 attempt(s) left. processing task id 441 item b04b3218 with 83695 events completed on d12chas564.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 132.0 MB, disk 330.0 MB, gpus: 0.0, runtime 293.579679 output: Using /cvmfs certificates. 2021-10-02 22:47:18 python_package_run(25803): setting up conda environment at /var/condor/execute/dir_23235/worker-230547-23244/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:18 python_package_run(25803): waiting for lock on directory /var/condor/execute/dir_23235/worker-230547-23244/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:18 python_package_run(25803): directory /var/condor/execute/dir_23235/worker-230547-23244/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:47:18 python_package_run(25803): activating environment 2021-10-02 22:47:18 python_package_run(25803): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_23235/worker-230547-23244/t.441/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_23235/worker-230547-23244/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_23235/worker-230547-23244/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_23235/worker-230547-23244/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_23235/worker-230547-23244/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_23235/worker-230547-23244/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_23235/worker-230547-23244/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_23235/worker-230547-23244/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_23235/worker-230547-23244/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttllNuNuJetNoHiggs_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_2353.root 2021-10-02 22:52:12 python_package_run(25803): command line exit code: 1 2021-10-02 22:52:12 python_package_run(25803): total runtime: 293.471 seconds task id 441 item b04b3218 failed: OUTPUT_MISS ('UL17_ttllJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttllNuNuJetNoHiggs_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_2353.root', 'Events', 0, 83695) b04b3218 without result. resubmitting b04b3218 partly as b04b3218. 1 attempt(s) left. processing task id 400 item e85b6676 with 119263 events completed on d12chas584.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 133.0 MB, disk 330.0 MB, gpus: 0.0, runtime 301.982394 output: Using /cvmfs certificates. 2021-10-02 22:47:10 python_package_run(18241): setting up conda environment at /var/condor/execute/dir_15741/worker-230547-15747/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:10 python_package_run(18241): waiting for lock on directory /var/condor/execute/dir_15741/worker-230547-15747/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:10 python_package_run(18241): directory /var/condor/execute/dir_15741/worker-230547-15747/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:47:10 python_package_run(18241): activating environment 2021-10-02 22:47:10 python_package_run(18241): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_15741/worker-230547-15747/t.400/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_15741/worker-230547-15747/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_15741/worker-230547-15747/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_15741/worker-230547-15747/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_15741/worker-230547-15747/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_15741/worker-230547-15747/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_15741/worker-230547-15747/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_15741/worker-230547-15747/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_15741/worker-230547-15747/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttlnuJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_13286.root 2021-10-02 22:52:12 python_package_run(18241): command line exit code: 1 2021-10-02 22:52:12 python_package_run(18241): total runtime: 301.863 seconds task id 400 item e85b6676 failed: OUTPUT_MISS ('UL17_ttlnuJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttlnuJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_13286.root', 'Events', 119263, 238526) e85b6676 without result. resubmitting e85b6676 partly as e85b6676. 1 attempt(s) left. processing task id 422 item 81e201b9 with 113449 events completed on d12chas580.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 133.0 MB, disk 330.0 MB, gpus: 0.0, runtime 301.415594 output: Using /cvmfs certificates. 2021-10-02 22:47:10 python_package_run(8909): setting up conda environment at /var/condor/execute/dir_5229/worker-230547-5310/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:10 python_package_run(8909): waiting for lock on directory /var/condor/execute/dir_5229/worker-230547-5310/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:11 python_package_run(8909): directory /var/condor/execute/dir_5229/worker-230547-5310/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:47:11 python_package_run(8909): activating environment 2021-10-02 22:47:11 python_package_run(8909): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_5229/worker-230547-5310/t.422/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_5229/worker-230547-5310/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_5229/worker-230547-5310/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_5229/worker-230547-5310/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_5229/worker-230547-5310/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_5229/worker-230547-5310/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_5229/worker-230547-5310/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_5229/worker-230547-5310/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_5229/worker-230547-5310/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttllNuNuJetNoHiggs_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_6723.root 2021-10-02 22:52:12 python_package_run(8909): command line exit code: 1 2021-10-02 22:52:12 python_package_run(8909): total runtime: 301.298 seconds task id 422 item 81e201b9 failed: OUTPUT_MISS ('UL17_ttllJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttllNuNuJetNoHiggs_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_6723.root', 'Events', 113449, 226898) 81e201b9 without result. resubmitting 81e201b9 partly as 81e201b9. 1 attempt(s) left. processing task id 463 item c2543901 with 83776 events completed on d12chas564.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 133.0 MB, disk 330.0 MB, gpus: 0.0, runtime 291.198977 output: Using /cvmfs certificates. 2021-10-02 22:47:21 python_package_run(26196): setting up conda environment at /var/condor/execute/dir_23235/worker-230547-23244/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:21 python_package_run(26196): waiting for lock on directory /var/condor/execute/dir_23235/worker-230547-23244/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:21 python_package_run(26196): directory /var/condor/execute/dir_23235/worker-230547-23244/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:47:21 python_package_run(26196): activating environment 2021-10-02 22:47:21 python_package_run(26196): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_23235/worker-230547-23244/t.463/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_23235/worker-230547-23244/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_23235/worker-230547-23244/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_23235/worker-230547-23244/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_23235/worker-230547-23244/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_23235/worker-230547-23244/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_23235/worker-230547-23244/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_23235/worker-230547-23244/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_23235/worker-230547-23244/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttllNuNuJetNoHiggs_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_10293.root 2021-10-02 22:52:12 python_package_run(26196): command line exit code: 1 2021-10-02 22:52:12 python_package_run(26196): total runtime: 291.088 seconds task id 463 item c2543901 failed: OUTPUT_MISS ('UL17_ttllJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttllNuNuJetNoHiggs_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_10293.root', 'Events', 83776, 167552) c2543901 without result. resubmitting c2543901 partly as c2543901. 1 attempt(s) left. processing task id 420 item 99de9cc3 with 83377 events completed on d12chas580.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 132.0 MB, disk 330.0 MB, gpus: 0.0, runtime 301.444768 output: Using /cvmfs certificates. 2021-10-02 22:47:10 python_package_run(8914): setting up conda environment at /var/condor/execute/dir_5229/worker-230547-5310/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:10 python_package_run(8914): waiting for lock on directory /var/condor/execute/dir_5229/worker-230547-5310/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:11 python_package_run(8914): directory /var/condor/execute/dir_5229/worker-230547-5310/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:47:11 python_package_run(8914): activating environment 2021-10-02 22:47:11 python_package_run(8914): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_5229/worker-230547-5310/t.420/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_5229/worker-230547-5310/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_5229/worker-230547-5310/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_5229/worker-230547-5310/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_5229/worker-230547-5310/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_5229/worker-230547-5310/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_5229/worker-230547-5310/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_5229/worker-230547-5310/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_5229/worker-230547-5310/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttllNuNuJetNoHiggs_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_7390.root 2021-10-02 22:52:12 python_package_run(8914): command line exit code: 1 2021-10-02 22:52:12 python_package_run(8914): total runtime: 301.331 seconds task id 420 item 99de9cc3 failed: OUTPUT_MISS ('UL17_ttllJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttllNuNuJetNoHiggs_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_7390.root', 'Events', 166758, 250135) 99de9cc3 without result. resubmitting 99de9cc3 partly as 99de9cc3. 1 attempt(s) left. processing task id 409 item f6cc5187 with 109272 events completed on d12chas546.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 132.0 MB, disk 330.0 MB, gpus: 0.0, runtime 302.068614 output: Using /cvmfs certificates. 2021-10-02 22:47:10 python_package_run(27555): setting up conda environment at /var/condor/execute/dir_24407/worker-230547-24421/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:10 python_package_run(27555): waiting for lock on directory /var/condor/execute/dir_24407/worker-230547-24421/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:10 python_package_run(27555): directory /var/condor/execute/dir_24407/worker-230547-24421/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:47:10 python_package_run(27555): activating environment 2021-10-02 22:47:10 python_package_run(27555): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_24407/worker-230547-24421/t.409/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_24407/worker-230547-24421/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_24407/worker-230547-24421/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_24407/worker-230547-24421/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_24407/worker-230547-24421/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_24407/worker-230547-24421/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_24407/worker-230547-24421/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_24407/worker-230547-24421/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_24407/worker-230547-24421/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttlnuJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_10294.root 2021-10-02 22:52:12 python_package_run(27555): command line exit code: 1 2021-10-02 22:52:12 python_package_run(27555): total runtime: 301.960 seconds task id 409 item f6cc5187 failed: OUTPUT_MISS ('UL17_ttlnuJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttlnuJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_10294.root', 'Events', 0, 109272) f6cc5187 without result. resubmitting f6cc5187 partly as f6cc5187. 1 attempt(s) left. processing task id 430 item cba5cfef with 83616 events completed on d12chas584.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 132.0 MB, disk 330.0 MB, gpus: 0.0, runtime 300.057452 output: Using /cvmfs certificates. 2021-10-02 22:47:12 python_package_run(18481): setting up conda environment at /var/condor/execute/dir_15741/worker-230547-15747/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:12 python_package_run(18481): waiting for lock on directory /var/condor/execute/dir_15741/worker-230547-15747/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:12 python_package_run(18481): directory /var/condor/execute/dir_15741/worker-230547-15747/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:47:12 python_package_run(18481): activating environment 2021-10-02 22:47:12 python_package_run(18481): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_15741/worker-230547-15747/t.430/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_15741/worker-230547-15747/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_15741/worker-230547-15747/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_15741/worker-230547-15747/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_15741/worker-230547-15747/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_15741/worker-230547-15747/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_15741/worker-230547-15747/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_15741/worker-230547-15747/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_15741/worker-230547-15747/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttllNuNuJetNoHiggs_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_4908.root 2021-10-02 22:52:12 python_package_run(18481): command line exit code: 1 2021-10-02 22:52:12 python_package_run(18481): total runtime: 299.948 seconds task id 430 item cba5cfef failed: OUTPUT_MISS ('UL17_ttllJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttllNuNuJetNoHiggs_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_4908.root', 'Events', 83616, 167232) cba5cfef without result. resubmitting cba5cfef partly as cba5cfef. 1 attempt(s) left. processing task id 447 item f8fdfac1 with 117814 events completed on d12chas564.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 132.0 MB, disk 330.0 MB, gpus: 0.0, runtime 293.330312 output: Using /cvmfs certificates. 2021-10-02 22:47:19 python_package_run(25976): setting up conda environment at /var/condor/execute/dir_23236/worker-230547-23258/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:19 python_package_run(25976): waiting for lock on directory /var/condor/execute/dir_23236/worker-230547-23258/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:19 python_package_run(25976): directory /var/condor/execute/dir_23236/worker-230547-23258/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:47:19 python_package_run(25976): activating environment 2021-10-02 22:47:19 python_package_run(25976): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_23236/worker-230547-23258/t.447/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_23236/worker-230547-23258/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_23236/worker-230547-23258/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_23236/worker-230547-23258/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_23236/worker-230547-23258/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_23236/worker-230547-23258/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_23236/worker-230547-23258/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_23236/worker-230547-23258/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_23236/worker-230547-23258/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttllNuNuJetNoHiggs_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_16901.root 2021-10-02 22:52:12 python_package_run(25976): command line exit code: 1 2021-10-02 22:52:12 python_package_run(25976): total runtime: 293.236 seconds task id 447 item f8fdfac1 failed: OUTPUT_MISS ('UL17_ttllJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttllNuNuJetNoHiggs_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_16901.root', 'Events', 117814, 235628) f8fdfac1 without result. resubmitting f8fdfac1 partly as f8fdfac1. 1 attempt(s) left. processing task id 454 item 513cd0fb with 83657 events completed on d12chas545.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.009, memory: 132.0 MB, disk 330.0 MB, gpus: 0.0, runtime 293.292589 output: Using /cvmfs certificates. 2021-10-02 22:47:19 python_package_run(28337): setting up conda environment at /var/condor/execute/dir_25722/worker-230547-25727/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:19 python_package_run(28337): waiting for lock on directory /var/condor/execute/dir_25722/worker-230547-25727/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:19 python_package_run(28337): directory /var/condor/execute/dir_25722/worker-230547-25727/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:47:19 python_package_run(28337): activating environment 2021-10-02 22:47:19 python_package_run(28337): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_25722/worker-230547-25727/t.454/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_25722/worker-230547-25727/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_25722/worker-230547-25727/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_25722/worker-230547-25727/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_25722/worker-230547-25727/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_25722/worker-230547-25727/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_25722/worker-230547-25727/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_25722/worker-230547-25727/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_25722/worker-230547-25727/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttllNuNuJetNoHiggs_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_12744.root 2021-10-02 22:52:12 python_package_run(28337): command line exit code: 1 2021-10-02 22:52:12 python_package_run(28337): total runtime: 293.173 seconds task id 454 item 513cd0fb failed: OUTPUT_MISS ('UL17_ttllJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttllNuNuJetNoHiggs_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_12744.root', 'Events', 0, 83657) 513cd0fb without result. resubmitting 513cd0fb partly as 513cd0fb. 1 attempt(s) left. processing task id 444 item 34c2dd95 with 103085 events completed on d12chas548.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 133.0 MB, disk 330.0 MB, gpus: 0.0, runtime 293.740391 output: Using /cvmfs certificates. 2021-10-02 22:47:18 python_package_run(25035): setting up conda environment at /var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:18 python_package_run(25035): waiting for lock on directory /var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:18 python_package_run(25035): directory /var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:47:18 python_package_run(25035): activating environment 2021-10-02 22:47:18 python_package_run(25035): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_24697/worker-230547-24702/t.444/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttllNuNuJetNoHiggs_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_17528.root 2021-10-02 22:52:12 python_package_run(25035): command line exit code: 1 2021-10-02 22:52:12 python_package_run(25035): total runtime: 293.637 seconds task id 444 item 34c2dd95 failed: OUTPUT_MISS ('UL17_ttllJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttllNuNuJetNoHiggs_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_17528.root', 'Events', 0, 103085) 34c2dd95 without result. resubmitting 34c2dd95 partly as 34c2dd95. 1 attempt(s) left. processing task id 448 item 8e8179e8 with 124612 events completed on d12chas564.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 133.0 MB, disk 330.0 MB, gpus: 0.0, runtime 293.383624 output: Using /cvmfs certificates. 2021-10-02 22:47:19 python_package_run(25978): setting up conda environment at /var/condor/execute/dir_23236/worker-230547-23258/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:19 python_package_run(25978): waiting for lock on directory /var/condor/execute/dir_23236/worker-230547-23258/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:19 python_package_run(25978): directory /var/condor/execute/dir_23236/worker-230547-23258/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:47:19 python_package_run(25978): activating environment 2021-10-02 22:47:19 python_package_run(25978): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_23236/worker-230547-23258/t.448/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_23236/worker-230547-23258/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_23236/worker-230547-23258/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_23236/worker-230547-23258/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_23236/worker-230547-23258/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_23236/worker-230547-23258/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_23236/worker-230547-23258/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_23236/worker-230547-23258/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_23236/worker-230547-23258/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttllNuNuJetNoHiggs_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_15782.root 2021-10-02 22:52:12 python_package_run(25978): command line exit code: 1 2021-10-02 22:52:12 python_package_run(25978): total runtime: 293.277 seconds task id 448 item 8e8179e8 failed: OUTPUT_MISS ('UL17_ttllJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttllNuNuJetNoHiggs_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_15782.root', 'Events', 0, 124612) 8e8179e8 without result. resubmitting 8e8179e8 partly as 8e8179e8. 1 attempt(s) left. processing task id 442 item 2a311b79 with 83695 events completed on d12chas580.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 133.0 MB, disk 330.0 MB, gpus: 0.0, runtime 293.770725 output: Using /cvmfs certificates. 2021-10-02 22:47:18 python_package_run(9429): setting up conda environment at /var/condor/execute/dir_5229/worker-230547-5310/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:18 python_package_run(9429): waiting for lock on directory /var/condor/execute/dir_5229/worker-230547-5310/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:18 python_package_run(9429): directory /var/condor/execute/dir_5229/worker-230547-5310/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:47:18 python_package_run(9429): activating environment 2021-10-02 22:47:18 python_package_run(9429): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_5229/worker-230547-5310/t.442/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_5229/worker-230547-5310/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_5229/worker-230547-5310/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_5229/worker-230547-5310/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_5229/worker-230547-5310/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_5229/worker-230547-5310/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_5229/worker-230547-5310/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_5229/worker-230547-5310/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_5229/worker-230547-5310/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttllNuNuJetNoHiggs_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_2353.root 2021-10-02 22:52:12 python_package_run(9429): command line exit code: 1 2021-10-02 22:52:12 python_package_run(9429): total runtime: 293.649 seconds task id 442 item 2a311b79 failed: OUTPUT_MISS ('UL17_ttllJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttllNuNuJetNoHiggs_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_2353.root', 'Events', 83695, 167390) 2a311b79 without result. resubmitting 2a311b79 partly as 2a311b79. 1 attempt(s) left. processing task id 399 item 311a4535 with 119263 events completed on d12chas584.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 132.0 MB, disk 330.0 MB, gpus: 0.0, runtime 302.180038 output: Using /cvmfs certificates. 2021-10-02 22:47:10 python_package_run(18242): setting up conda environment at /var/condor/execute/dir_15741/worker-230547-15747/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:10 python_package_run(18242): waiting for lock on directory /var/condor/execute/dir_15741/worker-230547-15747/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:10 python_package_run(18242): directory /var/condor/execute/dir_15741/worker-230547-15747/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:47:10 python_package_run(18242): activating environment 2021-10-02 22:47:10 python_package_run(18242): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_15741/worker-230547-15747/t.399/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_15741/worker-230547-15747/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_15741/worker-230547-15747/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_15741/worker-230547-15747/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_15741/worker-230547-15747/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_15741/worker-230547-15747/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_15741/worker-230547-15747/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_15741/worker-230547-15747/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_15741/worker-230547-15747/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttlnuJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_13286.root 2021-10-02 22:52:12 python_package_run(18242): command line exit code: 1 2021-10-02 22:52:12 python_package_run(18242): total runtime: 302.065 seconds task id 399 item 311a4535 failed: OUTPUT_MISS ('UL17_ttlnuJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttlnuJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_13286.root', 'Events', 0, 119263) 311a4535 without result. resubmitting 311a4535 partly as 311a4535. 1 attempt(s) left. processing task id 443 item ec90fb76 with 83695 events completed on d12chas580.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.01, memory: 138.0 MB, disk 330.0 MB, gpus: 0.0, runtime 293.835881 output: Using /cvmfs certificates. 2021-10-02 22:47:18 python_package_run(9428): setting up conda environment at /var/condor/execute/dir_5229/worker-230547-5310/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:18 python_package_run(9428): waiting for lock on directory /var/condor/execute/dir_5229/worker-230547-5310/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:18 python_package_run(9428): directory /var/condor/execute/dir_5229/worker-230547-5310/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:47:18 python_package_run(9428): activating environment 2021-10-02 22:47:18 python_package_run(9428): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_5229/worker-230547-5310/t.443/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_5229/worker-230547-5310/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_5229/worker-230547-5310/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_5229/worker-230547-5310/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_5229/worker-230547-5310/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_5229/worker-230547-5310/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_5229/worker-230547-5310/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_5229/worker-230547-5310/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_5229/worker-230547-5310/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttllNuNuJetNoHiggs_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_2353.root 2021-10-02 22:52:12 python_package_run(9428): command line exit code: 1 2021-10-02 22:52:12 python_package_run(9428): total runtime: 293.717 seconds task id 443 item ec90fb76 failed: OUTPUT_MISS ('UL17_ttllJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttllNuNuJetNoHiggs_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_2353.root', 'Events', 167390, 251085) ec90fb76 without result. resubmitting ec90fb76 partly as ec90fb76. 1 attempt(s) left. processing task id 425 item 6430ab5b with 120609 events completed on d12chas564.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 132.0 MB, disk 330.0 MB, gpus: 0.0, runtime 293.842216 output: Using /cvmfs certificates. 2021-10-02 22:47:18 python_package_run(25805): setting up conda environment at /var/condor/execute/dir_23235/worker-230547-23244/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:18 python_package_run(25805): waiting for lock on directory /var/condor/execute/dir_23235/worker-230547-23244/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:18 python_package_run(25805): directory /var/condor/execute/dir_23235/worker-230547-23244/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:47:19 python_package_run(25805): activating environment 2021-10-02 22:47:19 python_package_run(25805): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_23235/worker-230547-23244/t.425/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_23235/worker-230547-23244/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_23235/worker-230547-23244/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_23235/worker-230547-23244/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_23235/worker-230547-23244/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_23235/worker-230547-23244/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_23235/worker-230547-23244/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_23235/worker-230547-23244/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_23235/worker-230547-23244/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttllNuNuJetNoHiggs_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_5732.root 2021-10-02 22:52:12 python_package_run(25805): command line exit code: 1 2021-10-02 22:52:12 python_package_run(25805): total runtime: 293.734 seconds task id 425 item 6430ab5b failed: OUTPUT_MISS ('UL17_ttllJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttllNuNuJetNoHiggs_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_5732.root', 'Events', 0, 120609) 6430ab5b without result. resubmitting 6430ab5b partly as 6430ab5b. 1 attempt(s) left. processing task id 397 item 77e7bf20 with 119579 events completed on d12chas584.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 132.0 MB, disk 330.0 MB, gpus: 0.0, runtime 302.956222 output: Using /cvmfs certificates. 2021-10-02 22:47:10 python_package_run(18244): setting up conda environment at /var/condor/execute/dir_15741/worker-230547-15747/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:10 python_package_run(18244): waiting for lock on directory /var/condor/execute/dir_15741/worker-230547-15747/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:10 python_package_run(18244): directory /var/condor/execute/dir_15741/worker-230547-15747/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:47:10 python_package_run(18244): activating environment 2021-10-02 22:47:10 python_package_run(18244): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_15741/worker-230547-15747/t.397/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_15741/worker-230547-15747/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_15741/worker-230547-15747/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_15741/worker-230547-15747/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_15741/worker-230547-15747/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_15741/worker-230547-15747/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_15741/worker-230547-15747/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_15741/worker-230547-15747/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_15741/worker-230547-15747/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttlnuJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_13961.root 2021-10-02 22:52:13 python_package_run(18244): command line exit code: 1 2021-10-02 22:52:13 python_package_run(18244): total runtime: 302.844 seconds task id 397 item 77e7bf20 failed: OUTPUT_MISS ('UL17_ttlnuJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttlnuJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_13961.root', 'Events', 0, 119579) 77e7bf20 without result. resubmitting 77e7bf20 partly as 77e7bf20. 1 attempt(s) left. processing task id 410 item e4a9c968 with 109272 events completed on d12chas546.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 136.0 MB, disk 330.0 MB, gpus: 0.0, runtime 302.979215 output: Using /cvmfs certificates. 2021-10-02 22:47:10 python_package_run(27554): setting up conda environment at /var/condor/execute/dir_24407/worker-230547-24421/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:10 python_package_run(27554): waiting for lock on directory /var/condor/execute/dir_24407/worker-230547-24421/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:10 python_package_run(27554): directory /var/condor/execute/dir_24407/worker-230547-24421/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:47:10 python_package_run(27554): activating environment 2021-10-02 22:47:10 python_package_run(27554): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_24407/worker-230547-24421/t.410/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_24407/worker-230547-24421/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_24407/worker-230547-24421/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_24407/worker-230547-24421/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_24407/worker-230547-24421/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_24407/worker-230547-24421/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_24407/worker-230547-24421/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_24407/worker-230547-24421/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_24407/worker-230547-24421/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttlnuJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_10294.root 2021-10-02 22:52:13 python_package_run(27554): command line exit code: 1 2021-10-02 22:52:13 python_package_run(27554): total runtime: 302.882 seconds task id 410 item e4a9c968 failed: OUTPUT_MISS ('UL17_ttlnuJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttlnuJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_10294.root', 'Events', 109272, 218544) e4a9c968 without result. resubmitting e4a9c968 partly as e4a9c968. 1 attempt(s) left. processing task id 401 item 909389a4 with 107933 events completed on d12chas546.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 134.0 MB, disk 330.0 MB, gpus: 0.0, runtime 303.085724 output: Using /cvmfs certificates. 2021-10-02 22:47:10 python_package_run(27553): setting up conda environment at /var/condor/execute/dir_24407/worker-230547-24421/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:10 python_package_run(27553): waiting for lock on directory /var/condor/execute/dir_24407/worker-230547-24421/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:10 python_package_run(27553): directory /var/condor/execute/dir_24407/worker-230547-24421/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:47:10 python_package_run(27553): activating environment 2021-10-02 22:47:10 python_package_run(27553): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_24407/worker-230547-24421/t.401/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_24407/worker-230547-24421/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_24407/worker-230547-24421/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_24407/worker-230547-24421/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_24407/worker-230547-24421/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_24407/worker-230547-24421/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_24407/worker-230547-24421/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_24407/worker-230547-24421/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_24407/worker-230547-24421/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttlnuJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_12639.root 2021-10-02 22:52:13 python_package_run(27553): command line exit code: 1 2021-10-02 22:52:13 python_package_run(27553): total runtime: 302.984 seconds task id 401 item 909389a4 failed: OUTPUT_MISS ('UL17_ttlnuJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttlnuJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_12639.root', 'Events', 0, 107933) 909389a4 without result. resubmitting 909389a4 partly as 909389a4. 1 attempt(s) left. processing task id 412 item ca160c19 with 116061 events completed on d12chas580.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 134.0 MB, disk 330.0 MB, gpus: 0.0, runtime 303.106452 output: Using /cvmfs certificates. 2021-10-02 22:47:10 python_package_run(8851): setting up conda environment at /var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:10 python_package_run(8851): waiting for lock on directory /var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:10 python_package_run(8851): directory /var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:47:10 python_package_run(8851): activating environment 2021-10-02 22:47:10 python_package_run(8851): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_5228/worker-230547-5303/t.412/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttllNuNuJetNoHiggs_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_9651.root 2021-10-02 22:52:13 python_package_run(8851): command line exit code: 1 2021-10-02 22:52:13 python_package_run(8851): total runtime: 302.989 seconds task id 412 item ca160c19 failed: OUTPUT_MISS ('UL17_ttllJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttllNuNuJetNoHiggs_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_9651.root', 'Events', 116061, 232122) ca160c19 without result. resubmitting ca160c19 partly as ca160c19. 1 attempt(s) left. processing task id 411 item 22d5ed52 with 116061 events completed on d12chas546.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 132.0 MB, disk 330.0 MB, gpus: 0.0, runtime 303.136914 output: Using /cvmfs certificates. 2021-10-02 22:47:10 python_package_run(27552): setting up conda environment at /var/condor/execute/dir_24407/worker-230547-24421/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:10 python_package_run(27552): waiting for lock on directory /var/condor/execute/dir_24407/worker-230547-24421/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:10 python_package_run(27552): directory /var/condor/execute/dir_24407/worker-230547-24421/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:47:10 python_package_run(27552): activating environment 2021-10-02 22:47:10 python_package_run(27552): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_24407/worker-230547-24421/t.411/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_24407/worker-230547-24421/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_24407/worker-230547-24421/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_24407/worker-230547-24421/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_24407/worker-230547-24421/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_24407/worker-230547-24421/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_24407/worker-230547-24421/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_24407/worker-230547-24421/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_24407/worker-230547-24421/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttllNuNuJetNoHiggs_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_9651.root 2021-10-02 22:52:13 python_package_run(27552): command line exit code: 1 2021-10-02 22:52:13 python_package_run(27552): total runtime: 303.041 seconds task id 411 item 22d5ed52 failed: OUTPUT_MISS ('UL17_ttllJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttllNuNuJetNoHiggs_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_9651.root', 'Events', 0, 116061) 22d5ed52 without result. resubmitting 22d5ed52 partly as 22d5ed52. 1 attempt(s) left. processing task id 396 item f765e3fa with 117615 events completed on d12chas584.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 134.0 MB, disk 330.0 MB, gpus: 0.0, runtime 303.356504 output: Using /cvmfs certificates. 2021-10-02 22:47:10 python_package_run(18245): setting up conda environment at /var/condor/execute/dir_15741/worker-230547-15747/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:10 python_package_run(18245): waiting for lock on directory /var/condor/execute/dir_15741/worker-230547-15747/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:10 python_package_run(18245): directory /var/condor/execute/dir_15741/worker-230547-15747/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:47:10 python_package_run(18245): activating environment 2021-10-02 22:47:10 python_package_run(18245): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_15741/worker-230547-15747/t.396/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_15741/worker-230547-15747/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_15741/worker-230547-15747/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_15741/worker-230547-15747/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_15741/worker-230547-15747/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_15741/worker-230547-15747/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_15741/worker-230547-15747/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_15741/worker-230547-15747/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_15741/worker-230547-15747/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttlnuJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_14544.root 2021-10-02 22:52:13 python_package_run(18245): command line exit code: 1 2021-10-02 22:52:13 python_package_run(18245): total runtime: 303.242 seconds task id 396 item f765e3fa failed: OUTPUT_MISS ('UL17_ttlnuJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttlnuJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_14544.root', 'Events', 117615, 235230) f765e3fa without result. resubmitting f765e3fa partly as f765e3fa. 1 attempt(s) left. processing task id 419 item d11433bf with 83379 events completed on d12chas580.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 138.0 MB, disk 330.0 MB, gpus: 0.0, runtime 303.279143 output: Using /cvmfs certificates. 2021-10-02 22:47:10 python_package_run(8915): setting up conda environment at /var/condor/execute/dir_5229/worker-230547-5310/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:10 python_package_run(8915): waiting for lock on directory /var/condor/execute/dir_5229/worker-230547-5310/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:11 python_package_run(8915): directory /var/condor/execute/dir_5229/worker-230547-5310/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:47:11 python_package_run(8915): activating environment 2021-10-02 22:47:11 python_package_run(8915): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_5229/worker-230547-5310/t.419/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_5229/worker-230547-5310/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_5229/worker-230547-5310/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_5229/worker-230547-5310/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_5229/worker-230547-5310/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_5229/worker-230547-5310/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_5229/worker-230547-5310/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_5229/worker-230547-5310/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_5229/worker-230547-5310/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttllNuNuJetNoHiggs_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_7390.root 2021-10-02 22:52:14 python_package_run(8915): command line exit code: 1 2021-10-02 22:52:14 python_package_run(8915): total runtime: 303.169 seconds task id 419 item d11433bf failed: OUTPUT_MISS ('UL17_ttllJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttllNuNuJetNoHiggs_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_7390.root', 'Events', 83379, 166758) d11433bf without result. resubmitting d11433bf partly as d11433bf. 1 attempt(s) left. processing task id 421 item f2c2b43a with 113449 events completed on d12chas580.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 132.0 MB, disk 330.0 MB, gpus: 0.0, runtime 303.495433 output: Using /cvmfs certificates. 2021-10-02 22:47:10 python_package_run(8911): setting up conda environment at /var/condor/execute/dir_5229/worker-230547-5310/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:10 python_package_run(8911): waiting for lock on directory /var/condor/execute/dir_5229/worker-230547-5310/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:11 python_package_run(8911): directory /var/condor/execute/dir_5229/worker-230547-5310/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:47:11 python_package_run(8911): activating environment 2021-10-02 22:47:11 python_package_run(8911): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_5229/worker-230547-5310/t.421/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_5229/worker-230547-5310/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_5229/worker-230547-5310/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_5229/worker-230547-5310/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_5229/worker-230547-5310/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_5229/worker-230547-5310/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_5229/worker-230547-5310/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_5229/worker-230547-5310/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_5229/worker-230547-5310/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttllNuNuJetNoHiggs_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_6723.root 2021-10-02 22:52:14 python_package_run(8911): command line exit code: 1 2021-10-02 22:52:14 python_package_run(8911): total runtime: 303.382 seconds task id 421 item f2c2b43a failed: OUTPUT_MISS ('UL17_ttllJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttllNuNuJetNoHiggs_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_6723.root', 'Events', 0, 113449) f2c2b43a without result. resubmitting f2c2b43a partly as f2c2b43a. 1 attempt(s) left. processing task id 418 item cfec7462 with 83379 events completed on d12chas580.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 140.0 MB, disk 330.0 MB, gpus: 0.0, runtime 303.528613 output: Using /cvmfs certificates. 2021-10-02 22:47:10 python_package_run(8916): setting up conda environment at /var/condor/execute/dir_5229/worker-230547-5310/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:10 python_package_run(8916): waiting for lock on directory /var/condor/execute/dir_5229/worker-230547-5310/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:11 python_package_run(8916): directory /var/condor/execute/dir_5229/worker-230547-5310/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:47:11 python_package_run(8916): activating environment 2021-10-02 22:47:11 python_package_run(8916): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_5229/worker-230547-5310/t.418/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_5229/worker-230547-5310/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_5229/worker-230547-5310/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_5229/worker-230547-5310/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_5229/worker-230547-5310/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_5229/worker-230547-5310/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_5229/worker-230547-5310/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_5229/worker-230547-5310/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_5229/worker-230547-5310/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttllNuNuJetNoHiggs_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_7390.root 2021-10-02 22:52:14 python_package_run(8916): command line exit code: 1 2021-10-02 22:52:14 python_package_run(8916): total runtime: 303.418 seconds task id 418 item cfec7462 failed: OUTPUT_MISS ('UL17_ttllJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttllNuNuJetNoHiggs_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_7390.root', 'Events', 0, 83379) cfec7462 without result. resubmitting cfec7462 partly as cfec7462. 1 attempt(s) left. processing task id 415 item 292d5022 with 83815 events completed on d12chas580.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 142.0 MB, disk 330.0 MB, gpus: 0.0, runtime 303.570772 output: Using /cvmfs certificates. 2021-10-02 22:47:10 python_package_run(8921): setting up conda environment at /var/condor/execute/dir_5229/worker-230547-5310/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:10 python_package_run(8921): waiting for lock on directory /var/condor/execute/dir_5229/worker-230547-5310/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:11 python_package_run(8921): directory /var/condor/execute/dir_5229/worker-230547-5310/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:47:11 python_package_run(8921): activating environment 2021-10-02 22:47:11 python_package_run(8921): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_5229/worker-230547-5310/t.415/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_5229/worker-230547-5310/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_5229/worker-230547-5310/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_5229/worker-230547-5310/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_5229/worker-230547-5310/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_5229/worker-230547-5310/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_5229/worker-230547-5310/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_5229/worker-230547-5310/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_5229/worker-230547-5310/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttllNuNuJetNoHiggs_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_8064.root 2021-10-02 22:52:14 python_package_run(8921): command line exit code: 1 2021-10-02 22:52:14 python_package_run(8921): total runtime: 303.462 seconds task id 415 item 292d5022 failed: OUTPUT_MISS ('UL17_ttllJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttllNuNuJetNoHiggs_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_8064.root', 'Events', 0, 83815) 292d5022 without result. resubmitting 292d5022 partly as 292d5022. 1 attempt(s) left. processing task id 431 item b860bd22 with 83614 events completed on d12chas580.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 138.0 MB, disk 330.0 MB, gpus: 0.0, runtime 302.953013 output: Using /cvmfs certificates. 2021-10-02 22:47:13 python_package_run(9312): setting up conda environment at /var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:13 python_package_run(9312): waiting for lock on directory /var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:13 python_package_run(9312): directory /var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:47:13 python_package_run(9312): activating environment 2021-10-02 22:47:13 python_package_run(9312): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_5228/worker-230547-5303/t.431/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttllNuNuJetNoHiggs_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_4908.root 2021-10-02 22:52:16 python_package_run(9312): command line exit code: 1 2021-10-02 22:52:16 python_package_run(9312): total runtime: 302.846 seconds task id 431 item b860bd22 failed: OUTPUT_MISS ('UL17_ttllJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttllNuNuJetNoHiggs_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_4908.root', 'Events', 167232, 250846) b860bd22 without result. resubmitting b860bd22 partly as b860bd22. 1 attempt(s) left. processing task id 426 item cf5b0133 with 120608 events completed on d12chas546.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 134.0 MB, disk 330.0 MB, gpus: 0.0, runtime 303.163794 output: Using /cvmfs certificates. 2021-10-02 22:47:13 python_package_run(27782): setting up conda environment at /var/condor/execute/dir_24407/worker-230547-24421/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:13 python_package_run(27782): waiting for lock on directory /var/condor/execute/dir_24407/worker-230547-24421/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:13 python_package_run(27782): directory /var/condor/execute/dir_24407/worker-230547-24421/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:47:13 python_package_run(27782): activating environment 2021-10-02 22:47:13 python_package_run(27782): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_24407/worker-230547-24421/t.426/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_24407/worker-230547-24421/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_24407/worker-230547-24421/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_24407/worker-230547-24421/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_24407/worker-230547-24421/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_24407/worker-230547-24421/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_24407/worker-230547-24421/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_24407/worker-230547-24421/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_24407/worker-230547-24421/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttllNuNuJetNoHiggs_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_5732.root 2021-10-02 22:52:16 python_package_run(27782): command line exit code: 1 2021-10-02 22:52:16 python_package_run(27782): total runtime: 303.048 seconds task id 426 item cf5b0133 failed: OUTPUT_MISS ('UL17_ttllJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttllNuNuJetNoHiggs_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_5732.root', 'Events', 120609, 241217) cf5b0133 without result. resubmitting cf5b0133 partly as cf5b0133. 1 attempt(s) left. processing task id 429 item e98a8d57 with 83616 events completed on d12chas546.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 134.0 MB, disk 330.0 MB, gpus: 0.0, runtime 303.269918 output: Using /cvmfs certificates. 2021-10-02 22:47:13 python_package_run(27779): setting up conda environment at /var/condor/execute/dir_24407/worker-230547-24421/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:13 python_package_run(27779): waiting for lock on directory /var/condor/execute/dir_24407/worker-230547-24421/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:13 python_package_run(27779): directory /var/condor/execute/dir_24407/worker-230547-24421/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:47:13 python_package_run(27779): activating environment 2021-10-02 22:47:13 python_package_run(27779): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_24407/worker-230547-24421/t.429/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_24407/worker-230547-24421/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_24407/worker-230547-24421/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_24407/worker-230547-24421/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_24407/worker-230547-24421/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_24407/worker-230547-24421/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_24407/worker-230547-24421/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_24407/worker-230547-24421/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_24407/worker-230547-24421/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttllNuNuJetNoHiggs_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_4908.root 2021-10-02 22:52:16 python_package_run(27779): command line exit code: 1 2021-10-02 22:52:16 python_package_run(27779): total runtime: 303.153 seconds task id 429 item e98a8d57 failed: OUTPUT_MISS ('UL17_ttllJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttllNuNuJetNoHiggs_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_4908.root', 'Events', 0, 83616) e98a8d57 without result. resubmitting e98a8d57 partly as e98a8d57. 1 attempt(s) left. processing task id 427 item 4f82ab46 with 119308 events completed on d12chas546.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 133.0 MB, disk 330.0 MB, gpus: 0.0, runtime 303.325548 output: Using /cvmfs certificates. 2021-10-02 22:47:13 python_package_run(27781): setting up conda environment at /var/condor/execute/dir_24407/worker-230547-24421/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:13 python_package_run(27781): waiting for lock on directory /var/condor/execute/dir_24407/worker-230547-24421/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:13 python_package_run(27781): directory /var/condor/execute/dir_24407/worker-230547-24421/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:47:13 python_package_run(27781): activating environment 2021-10-02 22:47:13 python_package_run(27781): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_24407/worker-230547-24421/t.427/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_24407/worker-230547-24421/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_24407/worker-230547-24421/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_24407/worker-230547-24421/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_24407/worker-230547-24421/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_24407/worker-230547-24421/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_24407/worker-230547-24421/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_24407/worker-230547-24421/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_24407/worker-230547-24421/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttllNuNuJetNoHiggs_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_5335.root 2021-10-02 22:52:16 python_package_run(27781): command line exit code: 1 2021-10-02 22:52:16 python_package_run(27781): total runtime: 303.215 seconds task id 427 item 4f82ab46 failed: OUTPUT_MISS ('UL17_ttllJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttllNuNuJetNoHiggs_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_5335.root', 'Events', 0, 119308) 4f82ab46 without result. resubmitting 4f82ab46 partly as 4f82ab46. 1 attempt(s) left. processing task id 428 item 6e4f5411 with 119307 events completed on d12chas546.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 134.0 MB, disk 330.0 MB, gpus: 0.0, runtime 303.349246 output: Using /cvmfs certificates. 2021-10-02 22:47:13 python_package_run(27780): setting up conda environment at /var/condor/execute/dir_24407/worker-230547-24421/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:13 python_package_run(27780): waiting for lock on directory /var/condor/execute/dir_24407/worker-230547-24421/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:13 python_package_run(27780): directory /var/condor/execute/dir_24407/worker-230547-24421/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:47:13 python_package_run(27780): activating environment 2021-10-02 22:47:13 python_package_run(27780): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_24407/worker-230547-24421/t.428/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_24407/worker-230547-24421/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_24407/worker-230547-24421/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_24407/worker-230547-24421/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_24407/worker-230547-24421/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_24407/worker-230547-24421/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_24407/worker-230547-24421/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_24407/worker-230547-24421/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_24407/worker-230547-24421/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttllNuNuJetNoHiggs_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_5335.root 2021-10-02 22:52:16 python_package_run(27780): command line exit code: 1 2021-10-02 22:52:16 python_package_run(27780): total runtime: 303.236 seconds task id 428 item 6e4f5411 failed: OUTPUT_MISS ('UL17_ttllJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttllNuNuJetNoHiggs_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_5335.root', 'Events', 119308, 238615) 6e4f5411 without result. resubmitting 6e4f5411 partly as 6e4f5411. 1 attempt(s) left. processing task id 434 item 11dc2e66 with 83499 events completed on d12chas545.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 134.0 MB, disk 330.0 MB, gpus: 0.0, runtime 303.006684 output: Using /cvmfs certificates. 2021-10-02 22:47:14 python_package_run(28005): setting up conda environment at /var/condor/execute/dir_25722/worker-230547-25727/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:14 python_package_run(28005): waiting for lock on directory /var/condor/execute/dir_25722/worker-230547-25727/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:14 python_package_run(28005): directory /var/condor/execute/dir_25722/worker-230547-25727/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:47:14 python_package_run(28005): activating environment 2021-10-02 22:47:14 python_package_run(28005): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_25722/worker-230547-25727/t.434/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_25722/worker-230547-25727/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_25722/worker-230547-25727/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_25722/worker-230547-25727/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_25722/worker-230547-25727/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_25722/worker-230547-25727/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_25722/worker-230547-25727/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_25722/worker-230547-25727/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_25722/worker-230547-25727/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttllNuNuJetNoHiggs_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_4282.root 2021-10-02 22:52:17 python_package_run(28005): command line exit code: 1 2021-10-02 22:52:17 python_package_run(28005): total runtime: 302.889 seconds task id 434 item 11dc2e66 failed: OUTPUT_MISS ('UL17_ttllJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttllNuNuJetNoHiggs_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_4282.root', 'Events', 166998, 250497) 11dc2e66 without result. resubmitting 11dc2e66 partly as 11dc2e66. 1 attempt(s) left. processing task id 435 item 71d1f727 with 83494 events completed on d12chas545.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 133.0 MB, disk 330.0 MB, gpus: 0.0, runtime 303.11145 output: Using /cvmfs certificates. 2021-10-02 22:47:14 python_package_run(28003): setting up conda environment at /var/condor/execute/dir_25722/worker-230547-25727/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:14 python_package_run(28003): waiting for lock on directory /var/condor/execute/dir_25722/worker-230547-25727/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:14 python_package_run(28003): directory /var/condor/execute/dir_25722/worker-230547-25727/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:47:14 python_package_run(28003): activating environment 2021-10-02 22:47:14 python_package_run(28003): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_25722/worker-230547-25727/t.435/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_25722/worker-230547-25727/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_25722/worker-230547-25727/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_25722/worker-230547-25727/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_25722/worker-230547-25727/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_25722/worker-230547-25727/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_25722/worker-230547-25727/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_25722/worker-230547-25727/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_25722/worker-230547-25727/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttllNuNuJetNoHiggs_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_3824.root 2021-10-02 22:52:17 python_package_run(28003): command line exit code: 1 2021-10-02 22:52:17 python_package_run(28003): total runtime: 302.995 seconds task id 435 item 71d1f727 failed: OUTPUT_MISS ('UL17_ttllJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttllNuNuJetNoHiggs_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_3824.root', 'Events', 0, 83494) 71d1f727 without result. resubmitting 71d1f727 partly as 71d1f727. 1 attempt(s) left. processing task id 436 item 082610bb with 83494 events completed on d12chas545.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 134.0 MB, disk 330.0 MB, gpus: 0.0, runtime 303.119363 output: Using /cvmfs certificates. 2021-10-02 22:47:14 python_package_run(28002): setting up conda environment at /var/condor/execute/dir_25722/worker-230547-25727/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:14 python_package_run(28002): waiting for lock on directory /var/condor/execute/dir_25722/worker-230547-25727/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:14 python_package_run(28002): directory /var/condor/execute/dir_25722/worker-230547-25727/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:47:14 python_package_run(28002): activating environment 2021-10-02 22:47:14 python_package_run(28002): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_25722/worker-230547-25727/t.436/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_25722/worker-230547-25727/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_25722/worker-230547-25727/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_25722/worker-230547-25727/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_25722/worker-230547-25727/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_25722/worker-230547-25727/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_25722/worker-230547-25727/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_25722/worker-230547-25727/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_25722/worker-230547-25727/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttllNuNuJetNoHiggs_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_3824.root 2021-10-02 22:52:17 python_package_run(28002): command line exit code: 1 2021-10-02 22:52:17 python_package_run(28002): total runtime: 302.997 seconds task id 436 item 082610bb failed: OUTPUT_MISS ('UL17_ttllJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttllNuNuJetNoHiggs_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_3824.root', 'Events', 83494, 166988) 082610bb without result. resubmitting 082610bb partly as 082610bb. 1 attempt(s) left. processing task id 433 item cfe62b91 with 83499 events completed on d12chas545.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.012, memory: 132.0 MB, disk 330.0 MB, gpus: 0.0, runtime 303.586488 output: Using /cvmfs certificates. 2021-10-02 22:47:14 python_package_run(28004): setting up conda environment at /var/condor/execute/dir_25722/worker-230547-25727/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:14 python_package_run(28004): waiting for lock on directory /var/condor/execute/dir_25722/worker-230547-25727/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:14 python_package_run(28004): directory /var/condor/execute/dir_25722/worker-230547-25727/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:47:14 python_package_run(28004): activating environment 2021-10-02 22:47:14 python_package_run(28004): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_25722/worker-230547-25727/t.433/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_25722/worker-230547-25727/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_25722/worker-230547-25727/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_25722/worker-230547-25727/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_25722/worker-230547-25727/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_25722/worker-230547-25727/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_25722/worker-230547-25727/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_25722/worker-230547-25727/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_25722/worker-230547-25727/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttllNuNuJetNoHiggs_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_4282.root 2021-10-02 22:52:18 python_package_run(28004): command line exit code: 1 2021-10-02 22:52:18 python_package_run(28004): total runtime: 303.474 seconds task id 433 item cfe62b91 failed: OUTPUT_MISS ('UL17_ttllJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttllNuNuJetNoHiggs_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_4282.root', 'Events', 83499, 166998) cfe62b91 without result. resubmitting cfe62b91 partly as cfe62b91. 1 attempt(s) left. processing task id 440 item 49bf6c0d with 83833 events completed on d12chas546.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 134.0 MB, disk 330.0 MB, gpus: 0.0, runtime 302.935099 output: Using /cvmfs certificates. 2021-10-02 22:47:15 python_package_run(27976): setting up conda environment at /var/condor/execute/dir_24406/worker-230547-24415/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:15 python_package_run(27976): waiting for lock on directory /var/condor/execute/dir_24406/worker-230547-24415/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:16 python_package_run(27976): directory /var/condor/execute/dir_24406/worker-230547-24415/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:47:16 python_package_run(27976): activating environment 2021-10-02 22:47:16 python_package_run(27976): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_24406/worker-230547-24415/t.440/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_24406/worker-230547-24415/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_24406/worker-230547-24415/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_24406/worker-230547-24415/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_24406/worker-230547-24415/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_24406/worker-230547-24415/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_24406/worker-230547-24415/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_24406/worker-230547-24415/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_24406/worker-230547-24415/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttllNuNuJetNoHiggs_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_3624.root 2021-10-02 22:52:18 python_package_run(27976): command line exit code: 1 2021-10-02 22:52:18 python_package_run(27976): total runtime: 302.824 seconds task id 440 item 49bf6c0d failed: OUTPUT_MISS ('UL17_ttllJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttllNuNuJetNoHiggs_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_3624.root', 'Events', 167670, 251503) 49bf6c0d without result. resubmitting 49bf6c0d partly as 49bf6c0d. 1 attempt(s) left. processing task id 439 item fbbc3171 with 83835 events completed on d12chas546.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 134.0 MB, disk 330.0 MB, gpus: 0.0, runtime 303.031577 output: Using /cvmfs certificates. 2021-10-02 22:47:15 python_package_run(27977): setting up conda environment at /var/condor/execute/dir_24406/worker-230547-24415/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:15 python_package_run(27977): waiting for lock on directory /var/condor/execute/dir_24406/worker-230547-24415/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:15 python_package_run(27977): directory /var/condor/execute/dir_24406/worker-230547-24415/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:47:15 python_package_run(27977): activating environment 2021-10-02 22:47:15 python_package_run(27977): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_24406/worker-230547-24415/t.439/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_24406/worker-230547-24415/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_24406/worker-230547-24415/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_24406/worker-230547-24415/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_24406/worker-230547-24415/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_24406/worker-230547-24415/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_24406/worker-230547-24415/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_24406/worker-230547-24415/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_24406/worker-230547-24415/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttllNuNuJetNoHiggs_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_3624.root 2021-10-02 22:52:18 python_package_run(27977): command line exit code: 1 2021-10-02 22:52:18 python_package_run(27977): total runtime: 302.926 seconds task id 439 item fbbc3171 failed: OUTPUT_MISS ('UL17_ttllJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttllNuNuJetNoHiggs_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_3624.root', 'Events', 83835, 167670) fbbc3171 without result. resubmitting fbbc3171 partly as fbbc3171. 1 attempt(s) left. processing task id 438 item d5beb3df with 83835 events completed on d12chas546.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 132.0 MB, disk 330.0 MB, gpus: 0.0, runtime 303.166566 output: Using /cvmfs certificates. 2021-10-02 22:47:15 python_package_run(27978): setting up conda environment at /var/condor/execute/dir_24406/worker-230547-24415/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:15 python_package_run(27978): waiting for lock on directory /var/condor/execute/dir_24406/worker-230547-24415/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:15 python_package_run(27978): directory /var/condor/execute/dir_24406/worker-230547-24415/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:47:15 python_package_run(27978): activating environment 2021-10-02 22:47:15 python_package_run(27978): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_24406/worker-230547-24415/t.438/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_24406/worker-230547-24415/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_24406/worker-230547-24415/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_24406/worker-230547-24415/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_24406/worker-230547-24415/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_24406/worker-230547-24415/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_24406/worker-230547-24415/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_24406/worker-230547-24415/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_24406/worker-230547-24415/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttllNuNuJetNoHiggs_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_3624.root 2021-10-02 22:52:18 python_package_run(27978): command line exit code: 1 2021-10-02 22:52:18 python_package_run(27978): total runtime: 303.064 seconds task id 438 item d5beb3df failed: OUTPUT_MISS ('UL17_ttllJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttllNuNuJetNoHiggs_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_3624.root', 'Events', 0, 83835) d5beb3df without result. resubmitting d5beb3df partly as d5beb3df. 1 attempt(s) left. processing task id 473 item e335319a with 117734 events completed on d12chas564.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 132.0 MB, disk 330.0 MB, gpus: 0.0, runtime 290.731232 output: Using /cvmfs certificates. 2021-10-02 22:47:28 python_package_run(26478): setting up conda environment at /var/condor/execute/dir_23236/worker-230547-23258/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:28 python_package_run(26478): waiting for lock on directory /var/condor/execute/dir_23236/worker-230547-23258/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:28 python_package_run(26478): directory /var/condor/execute/dir_23236/worker-230547-23258/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:47:28 python_package_run(26478): activating environment 2021-10-02 22:47:28 python_package_run(26478): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_23236/worker-230547-23258/t.473/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_23236/worker-230547-23258/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_23236/worker-230547-23258/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_23236/worker-230547-23258/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_23236/worker-230547-23258/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_23236/worker-230547-23258/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_23236/worker-230547-23258/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_23236/worker-230547-23258/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_23236/worker-230547-23258/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttHJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_6435.root 2021-10-02 22:52:18 python_package_run(26478): command line exit code: 1 2021-10-02 22:52:18 python_package_run(26478): total runtime: 290.615 seconds task id 473 item e335319a failed: OUTPUT_MISS ('UL17_ttHJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttHJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_6435.root', 'Events', 117734, 235468) e335319a without result. resubmitting e335319a partly as e335319a. 1 attempt(s) left. processing task id 452 item 2d531ad7 with 114922 events completed on d12chas545.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.009, memory: 133.0 MB, disk 330.0 MB, gpus: 0.0, runtime 299.91884 output: Using /cvmfs certificates. 2021-10-02 22:47:19 python_package_run(28339): setting up conda environment at /var/condor/execute/dir_25722/worker-230547-25727/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:19 python_package_run(28339): waiting for lock on directory /var/condor/execute/dir_25722/worker-230547-25727/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:19 python_package_run(28339): directory /var/condor/execute/dir_25722/worker-230547-25727/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:47:19 python_package_run(28339): activating environment 2021-10-02 22:47:19 python_package_run(28339): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_25722/worker-230547-25727/t.452/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_25722/worker-230547-25727/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_25722/worker-230547-25727/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_25722/worker-230547-25727/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_25722/worker-230547-25727/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_25722/worker-230547-25727/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_25722/worker-230547-25727/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_25722/worker-230547-25727/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_25722/worker-230547-25727/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttllNuNuJetNoHiggs_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_13483.root 2021-10-02 22:52:18 python_package_run(28339): command line exit code: 1 2021-10-02 22:52:18 python_package_run(28339): total runtime: 299.802 seconds task id 452 item 2d531ad7 failed: OUTPUT_MISS ('UL17_ttllJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttllNuNuJetNoHiggs_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_13483.root', 'Events', 0, 114922) 2d531ad7 without result. resubmitting 2d531ad7 partly as 2d531ad7. 1 attempt(s) left. processing task id 474 item 937a33eb with 118057 events completed on d12chas564.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 133.0 MB, disk 330.0 MB, gpus: 0.0, runtime 290.777144 output: Using /cvmfs certificates. 2021-10-02 22:47:28 python_package_run(26477): setting up conda environment at /var/condor/execute/dir_23236/worker-230547-23258/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:28 python_package_run(26477): waiting for lock on directory /var/condor/execute/dir_23236/worker-230547-23258/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:28 python_package_run(26477): directory /var/condor/execute/dir_23236/worker-230547-23258/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:47:28 python_package_run(26477): activating environment 2021-10-02 22:47:28 python_package_run(26477): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_23236/worker-230547-23258/t.474/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_23236/worker-230547-23258/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_23236/worker-230547-23258/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_23236/worker-230547-23258/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_23236/worker-230547-23258/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_23236/worker-230547-23258/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_23236/worker-230547-23258/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_23236/worker-230547-23258/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_23236/worker-230547-23258/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttHJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_5690.root 2021-10-02 22:52:18 python_package_run(26477): command line exit code: 1 2021-10-02 22:52:18 python_package_run(26477): total runtime: 290.657 seconds task id 474 item 937a33eb failed: OUTPUT_MISS ('UL17_ttHJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttHJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_5690.root', 'Events', 0, 118057) 937a33eb without result. resubmitting 937a33eb partly as 937a33eb. 1 attempt(s) left. processing task id 462 item 847493d1 with 83776 events completed on d12chas564.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 134.0 MB, disk 330.0 MB, gpus: 0.0, runtime 297.85402 output: Using /cvmfs certificates. 2021-10-02 22:47:21 python_package_run(26197): setting up conda environment at /var/condor/execute/dir_23235/worker-230547-23244/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:21 python_package_run(26197): waiting for lock on directory /var/condor/execute/dir_23235/worker-230547-23244/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:21 python_package_run(26197): directory /var/condor/execute/dir_23235/worker-230547-23244/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:47:21 python_package_run(26197): activating environment 2021-10-02 22:47:21 python_package_run(26197): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_23235/worker-230547-23244/t.462/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_23235/worker-230547-23244/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_23235/worker-230547-23244/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_23235/worker-230547-23244/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_23235/worker-230547-23244/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_23235/worker-230547-23244/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_23235/worker-230547-23244/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_23235/worker-230547-23244/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_23235/worker-230547-23244/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttllNuNuJetNoHiggs_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_10293.root 2021-10-02 22:52:18 python_package_run(26197): command line exit code: 1 2021-10-02 22:52:18 python_package_run(26197): total runtime: 297.749 seconds task id 462 item 847493d1 failed: OUTPUT_MISS ('UL17_ttllJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttllNuNuJetNoHiggs_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_10293.root', 'Events', 0, 83776) 847493d1 without result. resubmitting 847493d1 partly as 847493d1. 1 attempt(s) left. processing task id 424 item b0a54562 with 117193 events completed on d12chas564.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 133.0 MB, disk 330.0 MB, gpus: 0.0, runtime 300.331274 output: Using /cvmfs certificates. 2021-10-02 22:47:18 python_package_run(25806): setting up conda environment at /var/condor/execute/dir_23235/worker-230547-23244/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:18 python_package_run(25806): waiting for lock on directory /var/condor/execute/dir_23235/worker-230547-23244/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:18 python_package_run(25806): directory /var/condor/execute/dir_23235/worker-230547-23244/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:47:18 python_package_run(25806): activating environment 2021-10-02 22:47:18 python_package_run(25806): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_23235/worker-230547-23244/t.424/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_23235/worker-230547-23244/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_23235/worker-230547-23244/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_23235/worker-230547-23244/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_23235/worker-230547-23244/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_23235/worker-230547-23244/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_23235/worker-230547-23244/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_23235/worker-230547-23244/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_23235/worker-230547-23244/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttllNuNuJetNoHiggs_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_6181.root 2021-10-02 22:52:18 python_package_run(25806): command line exit code: 1 2021-10-02 22:52:18 python_package_run(25806): total runtime: 300.229 seconds task id 424 item b0a54562 failed: OUTPUT_MISS ('UL17_ttllJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttllNuNuJetNoHiggs_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_6181.root', 'Events', 117193, 234386) b0a54562 without result. resubmitting b0a54562 partly as b0a54562. 1 attempt(s) left. processing task id 456 item e863d1d0 with 83657 events completed on d12chas553.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 133.0 MB, disk 330.0 MB, gpus: 0.0, runtime 297.871119 output: Using /cvmfs certificates. 2021-10-02 22:47:21 python_package_run(15686): setting up conda environment at /var/condor/execute/dir_12056/worker-230547-12074/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:21 python_package_run(15686): waiting for lock on directory /var/condor/execute/dir_12056/worker-230547-12074/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:21 python_package_run(15686): directory /var/condor/execute/dir_12056/worker-230547-12074/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:47:21 python_package_run(15686): activating environment 2021-10-02 22:47:21 python_package_run(15686): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_12056/worker-230547-12074/t.456/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_12056/worker-230547-12074/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_12056/worker-230547-12074/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_12056/worker-230547-12074/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_12056/worker-230547-12074/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_12056/worker-230547-12074/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_12056/worker-230547-12074/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_12056/worker-230547-12074/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_12056/worker-230547-12074/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttllNuNuJetNoHiggs_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_12744.root 2021-10-02 22:52:18 python_package_run(15686): command line exit code: 1 2021-10-02 22:52:18 python_package_run(15686): total runtime: 297.757 seconds task id 456 item e863d1d0 failed: OUTPUT_MISS ('UL17_ttllJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttllNuNuJetNoHiggs_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_12744.root', 'Events', 167314, 250971) e863d1d0 without result. resubmitting e863d1d0 partly as e863d1d0. 1 attempt(s) left. processing task id 469 item f38ff8ed with 109130 events completed on d12chas551.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.009, memory: 132.0 MB, disk 330.0 MB, gpus: 0.0, runtime 297.455611 output: Using /cvmfs certificates. 2021-10-02 22:47:21 python_package_run(9739): setting up conda environment at /var/condor/execute/dir_7659/worker-230547-7664/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:21 python_package_run(9739): waiting for lock on directory /var/condor/execute/dir_7659/worker-230547-7664/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:21 python_package_run(9739): directory /var/condor/execute/dir_7659/worker-230547-7664/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:47:21 python_package_run(9739): activating environment 2021-10-02 22:47:21 python_package_run(9739): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_7659/worker-230547-7664/t.469/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_7659/worker-230547-7664/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_7659/worker-230547-7664/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_7659/worker-230547-7664/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_7659/worker-230547-7664/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_7659/worker-230547-7664/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_7659/worker-230547-7664/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_7659/worker-230547-7664/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_7659/worker-230547-7664/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttHJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_8324.root 2021-10-02 22:52:18 python_package_run(9739): command line exit code: 1 2021-10-02 22:52:18 python_package_run(9739): total runtime: 297.332 seconds task id 469 item f38ff8ed failed: OUTPUT_MISS ('UL17_ttHJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttHJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_8324.root', 'Events', 109131, 218261) f38ff8ed without result. resubmitting f38ff8ed partly as f38ff8ed. 1 attempt(s) left. processing task id 446 item 6b31a190 with 117814 events completed on d12chas564.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 132.0 MB, disk 330.0 MB, gpus: 0.0, runtime 299.997963 output: Using /cvmfs certificates. 2021-10-02 22:47:19 python_package_run(25980): setting up conda environment at /var/condor/execute/dir_23236/worker-230547-23258/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:19 python_package_run(25980): waiting for lock on directory /var/condor/execute/dir_23236/worker-230547-23258/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:19 python_package_run(25980): directory /var/condor/execute/dir_23236/worker-230547-23258/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:47:19 python_package_run(25980): activating environment 2021-10-02 22:47:19 python_package_run(25980): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_23236/worker-230547-23258/t.446/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_23236/worker-230547-23258/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_23236/worker-230547-23258/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_23236/worker-230547-23258/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_23236/worker-230547-23258/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_23236/worker-230547-23258/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_23236/worker-230547-23258/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_23236/worker-230547-23258/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_23236/worker-230547-23258/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttllNuNuJetNoHiggs_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_16901.root 2021-10-02 22:52:18 python_package_run(25980): command line exit code: 1 2021-10-02 22:52:18 python_package_run(25980): total runtime: 299.897 seconds task id 446 item 6b31a190 failed: OUTPUT_MISS ('UL17_ttllJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttllNuNuJetNoHiggs_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_16901.root', 'Events', 0, 117814) 6b31a190 without result. resubmitting 6b31a190 partly as 6b31a190. 1 attempt(s) left. processing task id 471 item b57bb24a with 110778 events completed on d12chas551.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.009, memory: 132.0 MB, disk 330.0 MB, gpus: 0.0, runtime 297.552958 output: Using /cvmfs certificates. 2021-10-02 22:47:21 python_package_run(9737): setting up conda environment at /var/condor/execute/dir_7659/worker-230547-7664/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:21 python_package_run(9737): waiting for lock on directory /var/condor/execute/dir_7659/worker-230547-7664/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:21 python_package_run(9737): directory /var/condor/execute/dir_7659/worker-230547-7664/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:47:21 python_package_run(9737): activating environment 2021-10-02 22:47:21 python_package_run(9737): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_7659/worker-230547-7664/t.471/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_7659/worker-230547-7664/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_7659/worker-230547-7664/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_7659/worker-230547-7664/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_7659/worker-230547-7664/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_7659/worker-230547-7664/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_7659/worker-230547-7664/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_7659/worker-230547-7664/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_7659/worker-230547-7664/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttHJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_7295.root 2021-10-02 22:52:18 python_package_run(9737): command line exit code: 1 2021-10-02 22:52:19 python_package_run(9737): total runtime: 297.426 seconds task id 471 item b57bb24a failed: OUTPUT_MISS ('UL17_ttHJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttHJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_7295.root', 'Events', 110778, 221556) b57bb24a without result. resubmitting b57bb24a partly as b57bb24a. 1 attempt(s) left. processing task id 460 item 6a32324e with 119759 events completed on d12chas564.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 134.0 MB, disk 330.0 MB, gpus: 0.0, runtime 297.975929 output: Using /cvmfs certificates. 2021-10-02 22:47:21 python_package_run(26199): setting up conda environment at /var/condor/execute/dir_23235/worker-230547-23244/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:21 python_package_run(26199): waiting for lock on directory /var/condor/execute/dir_23235/worker-230547-23244/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:21 python_package_run(26199): directory /var/condor/execute/dir_23235/worker-230547-23244/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:47:21 python_package_run(26199): activating environment 2021-10-02 22:47:21 python_package_run(26199): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_23235/worker-230547-23244/t.460/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_23235/worker-230547-23244/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_23235/worker-230547-23244/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_23235/worker-230547-23244/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_23235/worker-230547-23244/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_23235/worker-230547-23244/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_23235/worker-230547-23244/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_23235/worker-230547-23244/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_23235/worker-230547-23244/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttllNuNuJetNoHiggs_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_11118.root 2021-10-02 22:52:19 python_package_run(26199): command line exit code: 1 2021-10-02 22:52:19 python_package_run(26199): total runtime: 297.872 seconds task id 460 item 6a32324e failed: OUTPUT_MISS ('UL17_ttllJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttllNuNuJetNoHiggs_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_11118.root', 'Events', 0, 119759) 6a32324e without result. resubmitting 6a32324e partly as 6a32324e. 1 attempt(s) left. processing task id 457 item 66b5e67f with 83670 events completed on d12chas553.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 134.0 MB, disk 330.0 MB, gpus: 0.0, runtime 297.980186 output: Using /cvmfs certificates. 2021-10-02 22:47:21 python_package_run(15685): setting up conda environment at /var/condor/execute/dir_12056/worker-230547-12074/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:21 python_package_run(15685): waiting for lock on directory /var/condor/execute/dir_12056/worker-230547-12074/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:21 python_package_run(15685): directory /var/condor/execute/dir_12056/worker-230547-12074/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:47:21 python_package_run(15685): activating environment 2021-10-02 22:47:21 python_package_run(15685): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_12056/worker-230547-12074/t.457/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_12056/worker-230547-12074/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_12056/worker-230547-12074/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_12056/worker-230547-12074/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_12056/worker-230547-12074/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_12056/worker-230547-12074/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_12056/worker-230547-12074/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_12056/worker-230547-12074/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_12056/worker-230547-12074/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttllNuNuJetNoHiggs_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_11917.root 2021-10-02 22:52:19 python_package_run(15685): command line exit code: 1 2021-10-02 22:52:19 python_package_run(15685): total runtime: 297.862 seconds task id 457 item 66b5e67f failed: OUTPUT_MISS ('UL17_ttllJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttllNuNuJetNoHiggs_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_11917.root', 'Events', 0, 83670) 66b5e67f without result. resubmitting 66b5e67f partly as 66b5e67f. 1 attempt(s) left. processing task id 458 item d9b24637 with 83670 events completed on d12chas553.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 134.0 MB, disk 330.0 MB, gpus: 0.0, runtime 297.981875 output: Using /cvmfs certificates. 2021-10-02 22:47:21 python_package_run(15684): setting up conda environment at /var/condor/execute/dir_12056/worker-230547-12074/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:21 python_package_run(15684): waiting for lock on directory /var/condor/execute/dir_12056/worker-230547-12074/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:21 python_package_run(15684): directory /var/condor/execute/dir_12056/worker-230547-12074/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:47:21 python_package_run(15684): activating environment 2021-10-02 22:47:21 python_package_run(15684): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_12056/worker-230547-12074/t.458/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_12056/worker-230547-12074/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_12056/worker-230547-12074/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_12056/worker-230547-12074/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_12056/worker-230547-12074/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_12056/worker-230547-12074/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_12056/worker-230547-12074/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_12056/worker-230547-12074/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_12056/worker-230547-12074/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttllNuNuJetNoHiggs_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_11917.root 2021-10-02 22:52:19 python_package_run(15684): command line exit code: 1 2021-10-02 22:52:19 python_package_run(15684): total runtime: 297.863 seconds task id 458 item d9b24637 failed: OUTPUT_MISS ('UL17_ttllJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttllNuNuJetNoHiggs_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_11917.root', 'Events', 83670, 167340) d9b24637 without result. resubmitting d9b24637 partly as d9b24637. 1 attempt(s) left. processing task id 423 item ae0aef7c with 117193 events completed on d12chas551.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.01, memory: 133.0 MB, disk 330.0 MB, gpus: 0.0, runtime 300.509434 output: Using /cvmfs certificates. 2021-10-02 22:47:18 python_package_run(9507): setting up conda environment at /var/condor/execute/dir_7659/worker-230547-7664/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:18 python_package_run(9507): waiting for lock on directory /var/condor/execute/dir_7659/worker-230547-7664/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:18 python_package_run(9507): directory /var/condor/execute/dir_7659/worker-230547-7664/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:47:18 python_package_run(9507): activating environment 2021-10-02 22:47:18 python_package_run(9507): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_7659/worker-230547-7664/t.423/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_7659/worker-230547-7664/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_7659/worker-230547-7664/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_7659/worker-230547-7664/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_7659/worker-230547-7664/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_7659/worker-230547-7664/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_7659/worker-230547-7664/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_7659/worker-230547-7664/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_7659/worker-230547-7664/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttllNuNuJetNoHiggs_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_6181.root 2021-10-02 22:52:19 python_package_run(9507): command line exit code: 1 2021-10-02 22:52:19 python_package_run(9507): total runtime: 300.388 seconds task id 423 item ae0aef7c failed: OUTPUT_MISS ('UL17_ttllJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttllNuNuJetNoHiggs_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_6181.root', 'Events', 0, 117193) ae0aef7c without result. resubmitting ae0aef7c partly as ae0aef7c. 1 attempt(s) left. processing task id 477 item 3809c2f8 with 118017 events completed on d12chas564.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 132.0 MB, disk 330.0 MB, gpus: 0.0, runtime 291.081483 output: Using /cvmfs certificates. 2021-10-02 22:47:28 python_package_run(26475): setting up conda environment at /var/condor/execute/dir_23236/worker-230547-23258/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:28 python_package_run(26475): waiting for lock on directory /var/condor/execute/dir_23236/worker-230547-23258/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:28 python_package_run(26475): directory /var/condor/execute/dir_23236/worker-230547-23258/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:47:28 python_package_run(26475): activating environment 2021-10-02 22:47:28 python_package_run(26475): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_23236/worker-230547-23258/t.477/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_23236/worker-230547-23258/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_23236/worker-230547-23258/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_23236/worker-230547-23258/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_23236/worker-230547-23258/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_23236/worker-230547-23258/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_23236/worker-230547-23258/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_23236/worker-230547-23258/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_23236/worker-230547-23258/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttHJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_4657.root 2021-10-02 22:52:19 python_package_run(26475): command line exit code: 1 2021-10-02 22:52:19 python_package_run(26475): total runtime: 290.969 seconds task id 477 item 3809c2f8 failed: OUTPUT_MISS ('UL17_ttHJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttHJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_4657.root', 'Events', 118018, 236035) 3809c2f8 without result. resubmitting 3809c2f8 partly as 3809c2f8. 1 attempt(s) left. processing task id 414 item eed4e174 with 118914 events completed on d12chas551.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 133.0 MB, disk 330.0 MB, gpus: 0.0, runtime 300.608703 output: Using /cvmfs certificates. 2021-10-02 22:47:18 python_package_run(9508): setting up conda environment at /var/condor/execute/dir_7659/worker-230547-7664/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:18 python_package_run(9508): waiting for lock on directory /var/condor/execute/dir_7659/worker-230547-7664/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:18 python_package_run(9508): directory /var/condor/execute/dir_7659/worker-230547-7664/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:47:18 python_package_run(9508): activating environment 2021-10-02 22:47:18 python_package_run(9508): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_7659/worker-230547-7664/t.414/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_7659/worker-230547-7664/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_7659/worker-230547-7664/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_7659/worker-230547-7664/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_7659/worker-230547-7664/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_7659/worker-230547-7664/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_7659/worker-230547-7664/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_7659/worker-230547-7664/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_7659/worker-230547-7664/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttllNuNuJetNoHiggs_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_8682.root 2021-10-02 22:52:19 python_package_run(9508): command line exit code: 1 2021-10-02 22:52:19 python_package_run(9508): total runtime: 300.492 seconds task id 414 item eed4e174 failed: OUTPUT_MISS ('UL17_ttllJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttllNuNuJetNoHiggs_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_8682.root', 'Events', 118915, 237829) eed4e174 without result. resubmitting eed4e174 partly as eed4e174. 1 attempt(s) left. processing task id 413 item e1aa23c1 with 118915 events completed on d12chas551.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.012, memory: 132.0 MB, disk 330.0 MB, gpus: 0.0, runtime 300.660975 output: Using /cvmfs certificates. 2021-10-02 22:47:18 python_package_run(9509): setting up conda environment at /var/condor/execute/dir_7659/worker-230547-7664/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:18 python_package_run(9509): waiting for lock on directory /var/condor/execute/dir_7659/worker-230547-7664/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:18 python_package_run(9509): directory /var/condor/execute/dir_7659/worker-230547-7664/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:47:18 python_package_run(9509): activating environment 2021-10-02 22:47:18 python_package_run(9509): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_7659/worker-230547-7664/t.413/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_7659/worker-230547-7664/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_7659/worker-230547-7664/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_7659/worker-230547-7664/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_7659/worker-230547-7664/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_7659/worker-230547-7664/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_7659/worker-230547-7664/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_7659/worker-230547-7664/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_7659/worker-230547-7664/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttllNuNuJetNoHiggs_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_8682.root 2021-10-02 22:52:19 python_package_run(9509): command line exit code: 1 2021-10-02 22:52:19 python_package_run(9509): total runtime: 300.548 seconds task id 413 item e1aa23c1 failed: OUTPUT_MISS ('UL17_ttllJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttllNuNuJetNoHiggs_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_8682.root', 'Events', 0, 118915) e1aa23c1 without result. resubmitting e1aa23c1 partly as e1aa23c1. 1 attempt(s) left. processing task id 432 item 646abe8b with 83499 events completed on d12chas564.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 132.0 MB, disk 330.0 MB, gpus: 0.0, runtime 300.801851 output: Using /cvmfs certificates. 2021-10-02 22:47:18 python_package_run(25804): setting up conda environment at /var/condor/execute/dir_23235/worker-230547-23244/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:18 python_package_run(25804): waiting for lock on directory /var/condor/execute/dir_23235/worker-230547-23244/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:18 python_package_run(25804): directory /var/condor/execute/dir_23235/worker-230547-23244/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:47:18 python_package_run(25804): activating environment 2021-10-02 22:47:18 python_package_run(25804): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_23235/worker-230547-23244/t.432/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_23235/worker-230547-23244/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_23235/worker-230547-23244/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_23235/worker-230547-23244/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_23235/worker-230547-23244/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_23235/worker-230547-23244/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_23235/worker-230547-23244/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_23235/worker-230547-23244/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_23235/worker-230547-23244/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttllNuNuJetNoHiggs_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_4282.root 2021-10-02 22:52:19 python_package_run(25804): command line exit code: 1 2021-10-02 22:52:19 python_package_run(25804): total runtime: 300.700 seconds task id 432 item 646abe8b failed: OUTPUT_MISS ('UL17_ttllJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttllNuNuJetNoHiggs_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_4282.root', 'Events', 0, 83499) 646abe8b without result. resubmitting 646abe8b partly as 646abe8b. 1 attempt(s) left. processing task id 470 item 52f122d1 with 110778 events completed on d12chas551.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 134.0 MB, disk 330.0 MB, gpus: 0.0, runtime 297.978506 output: Using /cvmfs certificates. 2021-10-02 22:47:21 python_package_run(9738): setting up conda environment at /var/condor/execute/dir_7659/worker-230547-7664/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:21 python_package_run(9738): waiting for lock on directory /var/condor/execute/dir_7659/worker-230547-7664/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:21 python_package_run(9738): directory /var/condor/execute/dir_7659/worker-230547-7664/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:47:21 python_package_run(9738): activating environment 2021-10-02 22:47:21 python_package_run(9738): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_7659/worker-230547-7664/t.470/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_7659/worker-230547-7664/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_7659/worker-230547-7664/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_7659/worker-230547-7664/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_7659/worker-230547-7664/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_7659/worker-230547-7664/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_7659/worker-230547-7664/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_7659/worker-230547-7664/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_7659/worker-230547-7664/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttHJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_7295.root 2021-10-02 22:52:19 python_package_run(9738): command line exit code: 1 2021-10-02 22:52:19 python_package_run(9738): total runtime: 297.859 seconds task id 470 item 52f122d1 failed: OUTPUT_MISS ('UL17_ttHJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttHJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_7295.root', 'Events', 0, 110778) 52f122d1 without result. resubmitting 52f122d1 partly as 52f122d1. 1 attempt(s) left. processing task id 461 item b6d356f3 with 119758 events completed on d12chas564.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 134.0 MB, disk 330.0 MB, gpus: 0.0, runtime 298.560147 output: Using /cvmfs certificates. 2021-10-02 22:47:21 python_package_run(26198): setting up conda environment at /var/condor/execute/dir_23235/worker-230547-23244/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:21 python_package_run(26198): waiting for lock on directory /var/condor/execute/dir_23235/worker-230547-23244/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:21 python_package_run(26198): directory /var/condor/execute/dir_23235/worker-230547-23244/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:47:21 python_package_run(26198): activating environment 2021-10-02 22:47:21 python_package_run(26198): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_23235/worker-230547-23244/t.461/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_23235/worker-230547-23244/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_23235/worker-230547-23244/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_23235/worker-230547-23244/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_23235/worker-230547-23244/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_23235/worker-230547-23244/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_23235/worker-230547-23244/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_23235/worker-230547-23244/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_23235/worker-230547-23244/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttllNuNuJetNoHiggs_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_11118.root 2021-10-02 22:52:19 python_package_run(26198): command line exit code: 1 2021-10-02 22:52:19 python_package_run(26198): total runtime: 298.457 seconds task id 461 item b6d356f3 failed: OUTPUT_MISS ('UL17_ttllJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttllNuNuJetNoHiggs_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_11118.root', 'Events', 119759, 239517) b6d356f3 without result. resubmitting b6d356f3 partly as b6d356f3. 1 attempt(s) left. processing task id 472 item 62891c70 with 117734 events completed on d12chas551.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 132.0 MB, disk 330.0 MB, gpus: 0.0, runtime 300.132963 output: Using /cvmfs certificates. 2021-10-02 22:47:21 python_package_run(9736): setting up conda environment at /var/condor/execute/dir_7659/worker-230547-7664/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:21 python_package_run(9736): waiting for lock on directory /var/condor/execute/dir_7659/worker-230547-7664/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:21 python_package_run(9736): directory /var/condor/execute/dir_7659/worker-230547-7664/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:47:21 python_package_run(9736): activating environment 2021-10-02 22:47:21 python_package_run(9736): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_7659/worker-230547-7664/t.472/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_7659/worker-230547-7664/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_7659/worker-230547-7664/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_7659/worker-230547-7664/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_7659/worker-230547-7664/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_7659/worker-230547-7664/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_7659/worker-230547-7664/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_7659/worker-230547-7664/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_7659/worker-230547-7664/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttHJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_6435.root 2021-10-02 22:52:21 python_package_run(9736): command line exit code: 1 2021-10-02 22:52:21 python_package_run(9736): total runtime: 300.004 seconds task id 472 item 62891c70 failed: OUTPUT_MISS ('UL17_ttHJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttHJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_6435.root', 'Events', 0, 117734) 62891c70 without result. resubmitting 62891c70 partly as 62891c70. 1 attempt(s) left. processing task id 459 item 7a97b475 with 83669 events completed on d12chas553.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 133.0 MB, disk 330.0 MB, gpus: 0.0, runtime 303.307332 output: Using /cvmfs certificates. 2021-10-02 22:47:21 python_package_run(15683): setting up conda environment at /var/condor/execute/dir_12056/worker-230547-12074/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:21 python_package_run(15683): waiting for lock on directory /var/condor/execute/dir_12056/worker-230547-12074/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:21 python_package_run(15683): directory /var/condor/execute/dir_12056/worker-230547-12074/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:47:21 python_package_run(15683): activating environment 2021-10-02 22:47:21 python_package_run(15683): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_12056/worker-230547-12074/t.459/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_12056/worker-230547-12074/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_12056/worker-230547-12074/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_12056/worker-230547-12074/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_12056/worker-230547-12074/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_12056/worker-230547-12074/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_12056/worker-230547-12074/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_12056/worker-230547-12074/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_12056/worker-230547-12074/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttllNuNuJetNoHiggs_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_11917.root 2021-10-02 22:52:24 python_package_run(15683): command line exit code: 1 2021-10-02 22:52:24 python_package_run(15683): total runtime: 303.183 seconds task id 459 item 7a97b475 failed: OUTPUT_MISS ('UL17_ttllJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttllNuNuJetNoHiggs_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_11917.root', 'Events', 167340, 251009) 7a97b475 without result. resubmitting 7a97b475 partly as 7a97b475. 1 attempt(s) left. processing task id 476 item 9abfe290 with 118018 events completed on d12chas546.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 132.0 MB, disk 330.0 MB, gpus: 0.0, runtime 303.160239 output: Using /cvmfs certificates. 2021-10-02 22:47:24 python_package_run(28231): setting up conda environment at /var/condor/execute/dir_24407/worker-230547-24421/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:24 python_package_run(28231): waiting for lock on directory /var/condor/execute/dir_24407/worker-230547-24421/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:25 python_package_run(28231): directory /var/condor/execute/dir_24407/worker-230547-24421/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:47:25 python_package_run(28231): activating environment 2021-10-02 22:47:25 python_package_run(28231): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_24407/worker-230547-24421/t.476/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_24407/worker-230547-24421/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_24407/worker-230547-24421/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_24407/worker-230547-24421/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_24407/worker-230547-24421/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_24407/worker-230547-24421/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_24407/worker-230547-24421/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_24407/worker-230547-24421/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_24407/worker-230547-24421/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttHJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_4657.root 2021-10-02 22:52:27 python_package_run(28231): command line exit code: 1 2021-10-02 22:52:27 python_package_run(28231): total runtime: 303.036 seconds task id 476 item 9abfe290 failed: OUTPUT_MISS ('UL17_ttHJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttHJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_4657.root', 'Events', 0, 118018) 9abfe290 without result. resubmitting 9abfe290 partly as 9abfe290. 1 attempt(s) left. processing task id 475 item 46292915 with 118056 events completed on d12chas564.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 134.0 MB, disk 330.0 MB, gpus: 0.0, runtime 303.145666 output: Using /cvmfs certificates. 2021-10-02 22:47:28 python_package_run(26476): setting up conda environment at /var/condor/execute/dir_23236/worker-230547-23258/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:28 python_package_run(26476): waiting for lock on directory /var/condor/execute/dir_23236/worker-230547-23258/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:28 python_package_run(26476): directory /var/condor/execute/dir_23236/worker-230547-23258/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:47:28 python_package_run(26476): activating environment 2021-10-02 22:47:28 python_package_run(26476): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_23236/worker-230547-23258/t.475/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_23236/worker-230547-23258/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_23236/worker-230547-23258/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_23236/worker-230547-23258/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_23236/worker-230547-23258/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_23236/worker-230547-23258/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_23236/worker-230547-23258/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_23236/worker-230547-23258/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_23236/worker-230547-23258/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttHJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_5690.root 2021-10-02 22:52:31 python_package_run(26476): command line exit code: 1 2021-10-02 22:52:31 python_package_run(26476): total runtime: 303.026 seconds task id 475 item 46292915 failed: OUTPUT_MISS ('UL17_ttHJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttHJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_5690.root', 'Events', 118057, 236113) 46292915 without result. resubmitting 46292915 partly as 46292915. 1 attempt(s) left. processing task id 481 item b6954b26 with 106863 events completed on d12chas564.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 134.0 MB, disk 330.0 MB, gpus: 0.0, runtime 302.864491 output: Using /cvmfs certificates. 2021-10-02 22:47:30 python_package_run(26690): setting up conda environment at /var/condor/execute/dir_23236/worker-230547-23258/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:30 python_package_run(26690): waiting for lock on directory /var/condor/execute/dir_23236/worker-230547-23258/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:30 python_package_run(26690): directory /var/condor/execute/dir_23236/worker-230547-23258/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:47:30 python_package_run(26690): activating environment 2021-10-02 22:47:30 python_package_run(26690): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_23236/worker-230547-23258/t.481/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_23236/worker-230547-23258/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_23236/worker-230547-23258/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_23236/worker-230547-23258/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_23236/worker-230547-23258/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_23236/worker-230547-23258/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_23236/worker-230547-23258/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_23236/worker-230547-23258/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_23236/worker-230547-23258/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttHJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_17069.root 2021-10-02 22:52:33 python_package_run(26690): command line exit code: 1 2021-10-02 22:52:33 python_package_run(26690): total runtime: 302.748 seconds task id 481 item b6954b26 failed: OUTPUT_MISS ('UL17_ttHJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttHJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_17069.root', 'Events', 0, 106863) b6954b26 without result. resubmitting b6954b26 partly as b6954b26. 1 attempt(s) left. processing task id 482 item 2c090759 with 106862 events completed on d12chas548.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 133.0 MB, disk 330.0 MB, gpus: 0.0, runtime 302.981735 output: Using /cvmfs certificates. 2021-10-02 22:47:31 python_package_run(25224): setting up conda environment at /var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:31 python_package_run(25224): waiting for lock on directory /var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:31 python_package_run(25224): directory /var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:47:31 python_package_run(25224): activating environment 2021-10-02 22:47:31 python_package_run(25224): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_24697/worker-230547-24702/t.482/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttHJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_17069.root 2021-10-02 22:52:34 python_package_run(25224): command line exit code: 1 2021-10-02 22:52:34 python_package_run(25224): total runtime: 302.861 seconds task id 482 item 2c090759 failed: OUTPUT_MISS ('UL17_ttHJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttHJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_17069.root', 'Events', 106863, 213725) 2c090759 without result. resubmitting 2c090759 partly as 2c090759. 1 attempt(s) left. processing task id 468 item f4920729 with 109131 events completed on d12chas546.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 133.0 MB, disk 330.0 MB, gpus: 0.0, runtime 296.989908 output: Using /cvmfs certificates. 2021-10-02 22:47:37 python_package_run(28317): setting up conda environment at /var/condor/execute/dir_24406/worker-230547-24415/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:37 python_package_run(28317): waiting for lock on directory /var/condor/execute/dir_24406/worker-230547-24415/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:37 python_package_run(28317): directory /var/condor/execute/dir_24406/worker-230547-24415/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:47:37 python_package_run(28317): activating environment 2021-10-02 22:47:37 python_package_run(28317): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_24406/worker-230547-24415/t.468/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_24406/worker-230547-24415/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_24406/worker-230547-24415/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_24406/worker-230547-24415/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_24406/worker-230547-24415/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_24406/worker-230547-24415/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_24406/worker-230547-24415/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_24406/worker-230547-24415/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_24406/worker-230547-24415/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttHJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_8324.root 2021-10-02 22:52:34 python_package_run(28317): command line exit code: 1 2021-10-02 22:52:34 python_package_run(28317): total runtime: 296.868 seconds task id 468 item f4920729 failed: OUTPUT_MISS ('UL17_ttHJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttHJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_8324.root', 'Events', 0, 109131) f4920729 without result. resubmitting f4920729 partly as f4920729. 1 attempt(s) left. processing task id 483 item a718850f with 115140 events completed on d12chas548.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 138.0 MB, disk 330.0 MB, gpus: 0.0, runtime 297.098482 output: Using /cvmfs certificates. 2021-10-02 22:47:37 python_package_run(25290): setting up conda environment at /var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:37 python_package_run(25290): waiting for lock on directory /var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:37 python_package_run(25290): directory /var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:47:37 python_package_run(25290): activating environment 2021-10-02 22:47:37 python_package_run(25290): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_24697/worker-230547-24702/t.483/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttHJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_16980.root 2021-10-02 22:52:34 python_package_run(25290): command line exit code: 1 2021-10-02 22:52:34 python_package_run(25290): total runtime: 296.984 seconds task id 483 item a718850f failed: OUTPUT_MISS ('UL17_ttHJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttHJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_16980.root', 'Events', 0, 115140) a718850f without result. resubmitting a718850f partly as a718850f. 1 attempt(s) left. processing task id 467 item afeb9b08 with 109534 events completed on d12chas546.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.009, memory: 133.0 MB, disk 330.0 MB, gpus: 0.0, runtime 297.75902 output: Using /cvmfs certificates. 2021-10-02 22:47:37 python_package_run(28318): setting up conda environment at /var/condor/execute/dir_24406/worker-230547-24415/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:37 python_package_run(28318): waiting for lock on directory /var/condor/execute/dir_24406/worker-230547-24415/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:38 python_package_run(28318): directory /var/condor/execute/dir_24406/worker-230547-24415/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:47:38 python_package_run(28318): activating environment 2021-10-02 22:47:38 python_package_run(28318): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_24406/worker-230547-24415/t.467/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_24406/worker-230547-24415/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_24406/worker-230547-24415/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_24406/worker-230547-24415/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_24406/worker-230547-24415/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_24406/worker-230547-24415/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_24406/worker-230547-24415/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_24406/worker-230547-24415/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_24406/worker-230547-24415/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttHJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_9503.root 2021-10-02 22:52:35 python_package_run(28318): command line exit code: 1 2021-10-02 22:52:35 python_package_run(28318): total runtime: 297.641 seconds task id 467 item afeb9b08 failed: OUTPUT_MISS ('UL17_ttHJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttHJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_9503.root', 'Events', 109535, 219069) afeb9b08 without result. resubmitting afeb9b08 partly as afeb9b08. 1 attempt(s) left. processing task id 478 item 7c4db65b with 117961 events completed on d12chas546.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 133.0 MB, disk 330.0 MB, gpus: 0.0, runtime 301.492891 output: Using /cvmfs certificates. 2021-10-02 22:47:37 python_package_run(28316): setting up conda environment at /var/condor/execute/dir_24406/worker-230547-24415/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:37 python_package_run(28316): waiting for lock on directory /var/condor/execute/dir_24406/worker-230547-24415/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:38 python_package_run(28316): directory /var/condor/execute/dir_24406/worker-230547-24415/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:47:38 python_package_run(28316): activating environment 2021-10-02 22:47:38 python_package_run(28316): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_24406/worker-230547-24415/t.478/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_24406/worker-230547-24415/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_24406/worker-230547-24415/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_24406/worker-230547-24415/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_24406/worker-230547-24415/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_24406/worker-230547-24415/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_24406/worker-230547-24415/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_24406/worker-230547-24415/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_24406/worker-230547-24415/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttHJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_3802.root 2021-10-02 22:52:39 python_package_run(28316): command line exit code: 1 2021-10-02 22:52:39 python_package_run(28316): total runtime: 301.372 seconds task id 478 item 7c4db65b failed: OUTPUT_MISS ('UL17_ttHJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttHJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_3802.root', 'Events', 0, 117961) 7c4db65b without result. resubmitting 7c4db65b partly as 7c4db65b. 1 attempt(s) left. processing task id 480 item a1b52282 with 79917 events completed on d12chas546.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.012, memory: 133.0 MB, disk 330.0 MB, gpus: 0.0, runtime 301.579897 output: Using /cvmfs certificates. 2021-10-02 22:47:37 python_package_run(28314): setting up conda environment at /var/condor/execute/dir_24406/worker-230547-24415/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:37 python_package_run(28314): waiting for lock on directory /var/condor/execute/dir_24406/worker-230547-24415/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:38 python_package_run(28314): directory /var/condor/execute/dir_24406/worker-230547-24415/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:47:38 python_package_run(28314): activating environment 2021-10-02 22:47:38 python_package_run(28314): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_24406/worker-230547-24415/t.480/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_24406/worker-230547-24415/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_24406/worker-230547-24415/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_24406/worker-230547-24415/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_24406/worker-230547-24415/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_24406/worker-230547-24415/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_24406/worker-230547-24415/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_24406/worker-230547-24415/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_24406/worker-230547-24415/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttHJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_17534.root 2021-10-02 22:52:39 python_package_run(28314): command line exit code: 1 2021-10-02 22:52:39 python_package_run(28314): total runtime: 301.451 seconds task id 480 item a1b52282 failed: OUTPUT_MISS ('UL17_ttHJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttHJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_17534.root', 'Events', 0, 79917) a1b52282 without result. resubmitting a1b52282 partly as a1b52282. 1 attempt(s) left. processing task id 490 item 45204ecc with 94500 events completed on d12chas547.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 134.0 MB, disk 330.0 MB, gpus: 0.0, runtime 302.78032 output: Using /cvmfs certificates. 2021-10-02 22:47:48 python_package_run(3410): setting up conda environment at /var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:48 python_package_run(3410): waiting for lock on directory /var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:47:48 python_package_run(3410): directory /var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:47:48 python_package_run(3410): activating environment 2021-10-02 22:47:48 python_package_run(3410): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_8819/worker-230547-8824/t.490/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_220.root 2021-10-02 22:52:51 python_package_run(3410): command line exit code: 1 2021-10-02 22:52:51 python_package_run(3410): total runtime: 302.664 seconds task id 490 item 45204ecc failed: OUTPUT_MISS ('UL17_tttt_b4', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_220.root', 'Events', 94500, 189000) 45204ecc without result. resubmitting 45204ecc partly as 45204ecc. 0 attempt(s) left. processing task id 527 item be614941 with 84334 events completed on d12chas577.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.014, memory: 135.0 MB, disk 330.0 MB, gpus: 0.0, runtime 191.627541 output: Using /cvmfs certificates. 2021-10-02 22:49:42 python_package_run(25837): setting up conda environment at /var/condor/execute/dir_22513/worker-230547-22528/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:49:42 python_package_run(25837): waiting for lock on directory /var/condor/execute/dir_22513/worker-230547-22528/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:49:42 python_package_run(25837): directory /var/condor/execute/dir_22513/worker-230547-22528/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:49:42 python_package_run(25837): activating environment 2021-10-02 22:49:42 python_package_run(25837): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_22513/worker-230547-22528/t.527/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_22513/worker-230547-22528/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1135, in _work_function factory = NanoEventsFactory.from_root( File "/var/condor/execute/dir_22513/worker-230547-22528/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/nanoevents/factory.py", line 97, in from_root tree = file[treepath] File "/var/condor/execute/dir_22513/worker-230547-22528/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 2081, in __getitem__ return self.key(where).get() File "/var/condor/execute/dir_22513/worker-230547-22528/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 2478, in get chunk, cursor = self.get_uncompressed_chunk_cursor() File "/var/condor/execute/dir_22513/worker-230547-22528/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 2530, in get_uncompressed_chunk_cursor chunk = self._file.chunk(data_start, data_stop) File "/var/condor/execute/dir_22513/worker-230547-22528/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 1207, in chunk return self._source.chunk(start, stop) File "/var/condor/execute/dir_22513/worker-230547-22528/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 305, in chunk data = self._resource.get(start, stop) File "/var/condor/execute/dir_22513/worker-230547-22528/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 168, in get self._xrd_error(status) File "/var/condor/execute/dir_22513/worker-230547-22528/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Socket timeout in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_tllq4fNoSchanWNoHiggs0p_all22WCsStartPtCheckV2dim6TopMay20GST_run0/NAOD-00000_5897.root 2021-10-02 22:52:53 python_package_run(25837): command line exit code: 1 2021-10-02 22:52:54 python_package_run(25837): total runtime: 191.549 seconds task id 527 item be614941 failed: OUTPUT_MISS ('UL17_tllq_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_tllq4fNoSchanWNoHiggs0p_all22WCsStartPtCheckV2dim6TopMay20GST_run0/NAOD-00000_5897.root', 'Events', 0, 84334) be614941 without result. resubmitting be614941 partly as be614941. 1 attempt(s) left. processing task id 517 item 485f2cd3 with 84166 events completed on d12chas587.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.295, memory: 1733.0 MB, disk 354.0 MB, gpus: 0.0, runtime 222.293712 processing task id 437 item 1a530202 with 83494 events completed on d12chas546.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.279, memory: 931.0 MB, disk 360.0 MB, gpus: 0.0, runtime 355.551256 submitted processing task id 665 item 215ae3cf, with 84000 events processing task id 417 item 3c3239aa with 83815 events completed on d12chas580.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.274, memory: 960.0 MB, disk 360.0 MB, gpus: 0.0, runtime 361.097551 submitted processing task id 666 item 886e5fe0, with 84000 events processing task id 416 item 0f734f89 with 83815 events completed on d12chas580.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.274, memory: 959.0 MB, disk 360.0 MB, gpus: 0.0, runtime 362.283292 submitted processing task id 667 item 20e357a8, with 84000 events submitted processing task id 668 item f417645f, with 84167 events processing task id 493 item 4f4fcba7 with 117840 events completed on d12chas548.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 138.0 MB, disk 330.0 MB, gpus: 0.0, runtime 293.706164 output: Using /cvmfs certificates. 2021-10-02 22:48:28 python_package_run(25593): setting up conda environment at /var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:48:28 python_package_run(25593): waiting for lock on directory /var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:48:28 python_package_run(25593): directory /var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:48:28 python_package_run(25593): activating environment 2021-10-02 22:48:28 python_package_run(25593): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_24697/worker-230547-24702/t.493/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttHJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_16900.root 2021-10-02 22:53:21 python_package_run(25593): command line exit code: 1 2021-10-02 22:53:21 python_package_run(25593): total runtime: 293.593 seconds task id 493 item 4f4fcba7 failed: OUTPUT_MISS ('UL17_ttHJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttHJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_16900.root', 'Events', 117841, 235681) 4f4fcba7 without result. resubmitting 4f4fcba7 partly as 4f4fcba7. 1 attempt(s) left. processing task id 488 item 76d4213f with 118117 events completed on d12chas552.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 133.0 MB, disk 330.0 MB, gpus: 0.0, runtime 299.60914 output: Using /cvmfs certificates. 2021-10-02 22:48:22 python_package_run(11763): setting up conda environment at /var/condor/execute/dir_8823/worker-230547-8828/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:48:22 python_package_run(11763): waiting for lock on directory /var/condor/execute/dir_8823/worker-230547-8828/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:48:22 python_package_run(11763): directory /var/condor/execute/dir_8823/worker-230547-8828/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:48:22 python_package_run(11763): activating environment 2021-10-02 22:48:22 python_package_run(11763): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_8823/worker-230547-8828/t.488/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_8823/worker-230547-8828/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_8823/worker-230547-8828/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_8823/worker-230547-8828/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_8823/worker-230547-8828/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_8823/worker-230547-8828/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_8823/worker-230547-8828/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_8823/worker-230547-8828/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_8823/worker-230547-8828/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttHJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_16914.root 2021-10-02 22:53:22 python_package_run(11763): command line exit code: 1 2021-10-02 22:53:22 python_package_run(11763): total runtime: 299.471 seconds task id 488 item 76d4213f failed: OUTPUT_MISS ('UL17_ttHJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttHJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_16914.root', 'Events', 118118, 236235) 76d4213f without result. resubmitting 76d4213f partly as 76d4213f. 1 attempt(s) left. processing task id 489 item 92b263e7 with 114381 events completed on qa-rtx6k-021.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.007, memory: 139.0 MB, disk 330.0 MB, gpus: 0.0, runtime 302.141304 output: Using /cvmfs certificates. 2021-10-02 22:48:20 python_package_run(159843): setting up conda environment at /var/condor/execute/dir_155377/worker-230547-155406/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:48:20 python_package_run(159843): waiting for lock on directory /var/condor/execute/dir_155377/worker-230547-155406/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:48:20 python_package_run(159843): directory /var/condor/execute/dir_155377/worker-230547-155406/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:48:20 python_package_run(159843): activating environment 2021-10-02 22:48:20 python_package_run(159843): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_155377/worker-230547-155406/t.489/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_155377/worker-230547-155406/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_155377/worker-230547-155406/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_155377/worker-230547-155406/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_155377/worker-230547-155406/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_155377/worker-230547-155406/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_155377/worker-230547-155406/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_155377/worker-230547-155406/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_155377/worker-230547-155406/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttHJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_16911.root 2021-10-02 22:53:22 python_package_run(159843): command line exit code: 1 2021-10-02 22:53:22 python_package_run(159843): total runtime: 302.041 seconds task id 489 item 92b263e7 failed: OUTPUT_MISS ('UL17_ttHJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttHJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_16911.root', 'Events', 0, 114381) 92b263e7 without result. resubmitting 92b263e7 partly as 92b263e7. 1 attempt(s) left. processing task id 510 item af1c16a1 with 111302 events completed on qa-rtx6k-021.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.258, memory: 1170.0 MB, disk 348.0 MB, gpus: 0.0, runtime 255.386385 processing task id 491 item c18ecd8e with 114381 events completed on d12chas548.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 133.0 MB, disk 330.0 MB, gpus: 0.0, runtime 303.181052 output: Using /cvmfs certificates. 2021-10-02 22:48:22 python_package_run(25524): setting up conda environment at /var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:48:22 python_package_run(25524): waiting for lock on directory /var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:48:22 python_package_run(25524): directory /var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:48:22 python_package_run(25524): activating environment 2021-10-02 22:48:22 python_package_run(25524): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_24697/worker-230547-24702/t.491/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttHJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_16911.root 2021-10-02 22:53:25 python_package_run(25524): command line exit code: 1 2021-10-02 22:53:25 python_package_run(25524): total runtime: 303.067 seconds task id 491 item c18ecd8e failed: OUTPUT_MISS ('UL17_ttHJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttHJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_16911.root', 'Events', 114381, 228762) c18ecd8e without result. resubmitting c18ecd8e partly as c18ecd8e. 1 attempt(s) left. processing task id 492 item dbd84d3a with 117841 events completed on d12chas582.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.008, memory: 133.0 MB, disk 330.0 MB, gpus: 0.0, runtime 301.01589 output: Using /cvmfs certificates. 2021-10-02 22:48:28 python_package_run(22850): setting up conda environment at /var/condor/execute/dir_16706/worker-230547-16711/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:48:28 python_package_run(22850): waiting for lock on directory /var/condor/execute/dir_16706/worker-230547-16711/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:48:28 python_package_run(22850): directory /var/condor/execute/dir_16706/worker-230547-16711/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:48:28 python_package_run(22850): activating environment 2021-10-02 22:48:28 python_package_run(22850): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_16706/worker-230547-16711/t.492/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_16706/worker-230547-16711/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_16706/worker-230547-16711/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_16706/worker-230547-16711/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_16706/worker-230547-16711/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_16706/worker-230547-16711/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_16706/worker-230547-16711/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_16706/worker-230547-16711/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_16706/worker-230547-16711/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttHJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_16900.root 2021-10-02 22:53:29 python_package_run(22850): command line exit code: 1 2021-10-02 22:53:29 python_package_run(22850): total runtime: 300.887 seconds task id 492 item dbd84d3a failed: OUTPUT_MISS ('UL17_ttHJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttHJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_16900.root', 'Events', 0, 117841) dbd84d3a without result. resubmitting dbd84d3a partly as dbd84d3a. 1 attempt(s) left. processing task id 495 item 66758e11 with 113672 events completed on d12chas545.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 134.0 MB, disk 330.0 MB, gpus: 0.0, runtime 302.977037 output: Using /cvmfs certificates. 2021-10-02 22:48:30 python_package_run(28618): setting up conda environment at /var/condor/execute/dir_25722/worker-230547-25727/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:48:30 python_package_run(28618): waiting for lock on directory /var/condor/execute/dir_25722/worker-230547-25727/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:48:30 python_package_run(28618): directory /var/condor/execute/dir_25722/worker-230547-25727/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:48:30 python_package_run(28618): activating environment 2021-10-02 22:48:30 python_package_run(28618): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_25722/worker-230547-25727/t.495/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_25722/worker-230547-25727/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_25722/worker-230547-25727/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_25722/worker-230547-25727/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_25722/worker-230547-25727/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_25722/worker-230547-25727/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_25722/worker-230547-25727/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_25722/worker-230547-25727/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_25722/worker-230547-25727/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttHJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_16782.root 2021-10-02 22:53:32 python_package_run(28618): command line exit code: 1 2021-10-02 22:53:32 python_package_run(28618): total runtime: 302.842 seconds task id 495 item 66758e11 failed: OUTPUT_MISS ('UL17_ttHJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttHJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_16782.root', 'Events', 113673, 227345) 66758e11 without result. resubmitting 66758e11 partly as 66758e11. 1 attempt(s) left. processing task id 496 item 41a614f5 with 111634 events completed on d12chas553.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.012, memory: 133.0 MB, disk 330.0 MB, gpus: 0.0, runtime 303.706512 output: Using /cvmfs certificates. 2021-10-02 22:48:31 python_package_run(16071): setting up conda environment at /var/condor/execute/dir_12055/worker-230547-12064/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:48:31 python_package_run(16071): waiting for lock on directory /var/condor/execute/dir_12055/worker-230547-12064/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:48:31 python_package_run(16071): directory /var/condor/execute/dir_12055/worker-230547-12064/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:48:31 python_package_run(16071): activating environment 2021-10-02 22:48:31 python_package_run(16071): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_12055/worker-230547-12064/t.496/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_12055/worker-230547-12064/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_12055/worker-230547-12064/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_12055/worker-230547-12064/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_12055/worker-230547-12064/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_12055/worker-230547-12064/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_12055/worker-230547-12064/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_12055/worker-230547-12064/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_12055/worker-230547-12064/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttHJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_16125.root 2021-10-02 22:53:35 python_package_run(16071): command line exit code: 1 2021-10-02 22:53:35 python_package_run(16071): total runtime: 303.579 seconds task id 496 item 41a614f5 failed: OUTPUT_MISS ('UL17_ttHJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttHJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_16125.root', 'Events', 0, 111634) 41a614f5 without result. resubmitting 41a614f5 partly as 41a614f5. 1 attempt(s) left. processing task id 497 item 7f44a48f with 111633 events completed on d12chas548.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 140.0 MB, disk 330.0 MB, gpus: 0.0, runtime 303.046768 output: Using /cvmfs certificates. 2021-10-02 22:48:34 python_package_run(25659): setting up conda environment at /var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:48:34 python_package_run(25659): waiting for lock on directory /var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:48:34 python_package_run(25659): directory /var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:48:34 python_package_run(25659): activating environment 2021-10-02 22:48:34 python_package_run(25659): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_24697/worker-230547-24702/t.497/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttHJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_16125.root 2021-10-02 22:53:37 python_package_run(25659): command line exit code: 1 2021-10-02 22:53:37 python_package_run(25659): total runtime: 302.932 seconds task id 497 item 7f44a48f failed: OUTPUT_MISS ('UL17_ttHJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttHJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_16125.root', 'Events', 111634, 223267) 7f44a48f without result. resubmitting 7f44a48f partly as 7f44a48f. 1 attempt(s) left. processing task id 498 item c36d2231 with 107092 events completed on d12chas547.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 134.0 MB, disk 330.0 MB, gpus: 0.0, runtime 302.68556 output: Using /cvmfs certificates. 2021-10-02 22:48:38 python_package_run(3717): setting up conda environment at /var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:48:38 python_package_run(3717): waiting for lock on directory /var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:48:38 python_package_run(3717): directory /var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:48:38 python_package_run(3717): activating environment 2021-10-02 22:48:38 python_package_run(3717): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_8819/worker-230547-8824/t.498/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttHJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_15478.root 2021-10-02 22:53:40 python_package_run(3717): command line exit code: 1 2021-10-02 22:53:40 python_package_run(3717): total runtime: 302.570 seconds task id 498 item c36d2231 failed: OUTPUT_MISS ('UL17_ttHJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttHJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_15478.root', 'Events', 0, 107092) c36d2231 without result. resubmitting c36d2231 partly as c36d2231. 1 attempt(s) left. processing task id 453 item bc5203f4 with 114922 events completed on d12chas545.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.353, memory: 1200.0 MB, disk 364.0 MB, gpus: 0.0, runtime 382.422585 submitted processing task id 678 item 3cd4a28f, with 84167 events processing task id 499 item a5e80bc0 with 107091 events completed on d12chas553.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 133.0 MB, disk 330.0 MB, gpus: 0.0, runtime 303.564464 output: Using /cvmfs certificates. 2021-10-02 22:48:38 python_package_run(16138): setting up conda environment at /var/condor/execute/dir_12055/worker-230547-12064/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:48:38 python_package_run(16138): waiting for lock on directory /var/condor/execute/dir_12055/worker-230547-12064/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:48:38 python_package_run(16138): directory /var/condor/execute/dir_12055/worker-230547-12064/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:48:38 python_package_run(16138): activating environment 2021-10-02 22:48:38 python_package_run(16138): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_12055/worker-230547-12064/t.499/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_12055/worker-230547-12064/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_12055/worker-230547-12064/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_12055/worker-230547-12064/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_12055/worker-230547-12064/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_12055/worker-230547-12064/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_12055/worker-230547-12064/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_12055/worker-230547-12064/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_12055/worker-230547-12064/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttHJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_15478.root 2021-10-02 22:53:42 python_package_run(16138): command line exit code: 1 2021-10-02 22:53:42 python_package_run(16138): total runtime: 303.445 seconds task id 499 item a5e80bc0 failed: OUTPUT_MISS ('UL17_ttHJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttHJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_15478.root', 'Events', 107092, 214183) a5e80bc0 without result. resubmitting a5e80bc0 partly as a5e80bc0. 1 attempt(s) left. processing task id 500 item 8f01a6db with 111956 events completed on d12chas547.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 133.0 MB, disk 330.0 MB, gpus: 0.0, runtime 302.984799 output: Using /cvmfs certificates. 2021-10-02 22:48:42 python_package_run(3781): setting up conda environment at /var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:48:42 python_package_run(3781): waiting for lock on directory /var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:48:42 python_package_run(3781): directory /var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:48:42 python_package_run(3781): activating environment 2021-10-02 22:48:42 python_package_run(3781): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_8819/worker-230547-8824/t.500/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttHJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_14737.root 2021-10-02 22:53:45 python_package_run(3781): command line exit code: 1 2021-10-02 22:53:45 python_package_run(3781): total runtime: 302.863 seconds task id 500 item 8f01a6db failed: OUTPUT_MISS ('UL17_ttHJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttHJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_14737.root', 'Events', 0, 111956) 8f01a6db without result. resubmitting 8f01a6db partly as 8f01a6db. 1 attempt(s) left. processing task id 479 item 1d227c07 with 117960 events completed on d12chas546.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.353, memory: 1235.0 MB, disk 349.0 MB, gpus: 0.0, runtime 371.963795 submitted processing task id 681 item 422af9e3, with 84166 events processing task id 501 item 5085a027 with 111956 events completed on qa-rtx6k-021.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.007, memory: 133.0 MB, disk 330.0 MB, gpus: 0.0, runtime 302.442391 output: Using /cvmfs certificates. 2021-10-02 22:48:50 python_package_run(159930): setting up conda environment at /var/condor/execute/dir_155377/worker-230547-155406/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:48:50 python_package_run(159930): waiting for lock on directory /var/condor/execute/dir_155377/worker-230547-155406/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:48:50 python_package_run(159930): directory /var/condor/execute/dir_155377/worker-230547-155406/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:48:50 python_package_run(159930): activating environment 2021-10-02 22:48:50 python_package_run(159930): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_155377/worker-230547-155406/t.501/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_155377/worker-230547-155406/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_155377/worker-230547-155406/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_155377/worker-230547-155406/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_155377/worker-230547-155406/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_155377/worker-230547-155406/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_155377/worker-230547-155406/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_155377/worker-230547-155406/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_155377/worker-230547-155406/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttHJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_14737.root 2021-10-02 22:53:52 python_package_run(159930): command line exit code: 1 2021-10-02 22:53:52 python_package_run(159930): total runtime: 302.340 seconds task id 501 item 5085a027 failed: OUTPUT_MISS ('UL17_ttHJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttHJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_14737.root', 'Events', 111956, 223912) 5085a027 without result. resubmitting 5085a027 partly as 5085a027. 1 attempt(s) left. processing task id 502 item 0ba00551 with 107703 events completed on d12chas584.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 134.0 MB, disk 330.0 MB, gpus: 0.0, runtime 302.921284 output: Using /cvmfs certificates. 2021-10-02 22:48:53 python_package_run(18770): setting up conda environment at /var/condor/execute/dir_15741/worker-230547-15747/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:48:53 python_package_run(18770): waiting for lock on directory /var/condor/execute/dir_15741/worker-230547-15747/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:48:53 python_package_run(18770): directory /var/condor/execute/dir_15741/worker-230547-15747/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:48:53 python_package_run(18770): activating environment 2021-10-02 22:48:53 python_package_run(18770): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_15741/worker-230547-15747/t.502/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_15741/worker-230547-15747/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_15741/worker-230547-15747/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_15741/worker-230547-15747/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_15741/worker-230547-15747/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_15741/worker-230547-15747/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_15741/worker-230547-15747/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_15741/worker-230547-15747/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_15741/worker-230547-15747/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttHJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_14060.root 2021-10-02 22:53:56 python_package_run(18770): command line exit code: 1 2021-10-02 22:53:56 python_package_run(18770): total runtime: 302.781 seconds task id 502 item 0ba00551 failed: OUTPUT_MISS ('UL17_ttHJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttHJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_14060.root', 'Events', 0, 107703) 0ba00551 without result. resubmitting 0ba00551 partly as 0ba00551. 1 attempt(s) left. submitted processing task id 684 item 27f14d17, with 84000 events processing task id 505 item 57fb30c9 with 111196 events completed on qa-rtx6k-021.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.007, memory: 133.0 MB, disk 330.0 MB, gpus: 0.0, runtime 301.197472 output: Using /cvmfs certificates. 2021-10-02 22:49:04 python_package_run(160009): setting up conda environment at /var/condor/execute/dir_155377/worker-230547-155406/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:49:04 python_package_run(160009): waiting for lock on directory /var/condor/execute/dir_155377/worker-230547-155406/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:49:04 python_package_run(160009): directory /var/condor/execute/dir_155377/worker-230547-155406/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:49:04 python_package_run(160009): activating environment 2021-10-02 22:49:04 python_package_run(160009): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_155377/worker-230547-155406/t.505/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_155377/worker-230547-155406/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_155377/worker-230547-155406/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_155377/worker-230547-155406/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_155377/worker-230547-155406/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_155377/worker-230547-155406/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_155377/worker-230547-155406/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_155377/worker-230547-155406/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_155377/worker-230547-155406/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttHJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_13390.root 2021-10-02 22:54:05 python_package_run(160009): command line exit code: 1 2021-10-02 22:54:05 python_package_run(160009): total runtime: 301.098 seconds task id 505 item 57fb30c9 failed: OUTPUT_MISS ('UL17_ttHJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttHJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_13390.root', 'Events', 0, 111196) 57fb30c9 without result. resubmitting 57fb30c9 partly as 57fb30c9. 1 attempt(s) left. processing task id 494 item a4d1e7a1 with 113673 events completed on d12chas551.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.009, memory: 134.0 MB, disk 330.0 MB, gpus: 0.0, runtime 302.8445 output: Using /cvmfs certificates. 2021-10-02 22:49:03 python_package_run(10148): setting up conda environment at /var/condor/execute/dir_7659/worker-230547-7664/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:49:03 python_package_run(10148): waiting for lock on directory /var/condor/execute/dir_7659/worker-230547-7664/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:49:03 python_package_run(10148): directory /var/condor/execute/dir_7659/worker-230547-7664/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:49:03 python_package_run(10148): activating environment 2021-10-02 22:49:03 python_package_run(10148): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_7659/worker-230547-7664/t.494/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_7659/worker-230547-7664/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_7659/worker-230547-7664/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_7659/worker-230547-7664/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_7659/worker-230547-7664/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_7659/worker-230547-7664/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_7659/worker-230547-7664/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_7659/worker-230547-7664/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_7659/worker-230547-7664/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttHJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_16782.root 2021-10-02 22:54:05 python_package_run(10148): command line exit code: 1 2021-10-02 22:54:05 python_package_run(10148): total runtime: 302.715 seconds task id 494 item a4d1e7a1 failed: OUTPUT_MISS ('UL17_ttHJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttHJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_16782.root', 'Events', 0, 113673) a4d1e7a1 without result. resubmitting a4d1e7a1 partly as a4d1e7a1. 1 attempt(s) left. processing task id 503 item 6894b4b3 with 107702 events completed on d12chas553.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 133.0 MB, disk 330.0 MB, gpus: 0.0, runtime 303.285216 output: Using /cvmfs certificates. 2021-10-02 22:49:03 python_package_run(16221): setting up conda environment at /var/condor/execute/dir_12055/worker-230547-12064/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:49:03 python_package_run(16221): waiting for lock on directory /var/condor/execute/dir_12055/worker-230547-12064/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:49:03 python_package_run(16221): directory /var/condor/execute/dir_12055/worker-230547-12064/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:49:03 python_package_run(16221): activating environment 2021-10-02 22:49:03 python_package_run(16221): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_12055/worker-230547-12064/t.503/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_12055/worker-230547-12064/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_12055/worker-230547-12064/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_12055/worker-230547-12064/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_12055/worker-230547-12064/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_12055/worker-230547-12064/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_12055/worker-230547-12064/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_12055/worker-230547-12064/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_12055/worker-230547-12064/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttHJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_14060.root 2021-10-02 22:54:06 python_package_run(16221): command line exit code: 1 2021-10-02 22:54:06 python_package_run(16221): total runtime: 303.168 seconds task id 503 item 6894b4b3 failed: OUTPUT_MISS ('UL17_ttHJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttHJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_14060.root', 'Events', 107703, 215405) 6894b4b3 without result. resubmitting 6894b4b3 partly as 6894b4b3. 1 attempt(s) left. processing task id 485 item 353e4b0a with 117970 events completed on qa-rtx6k-021.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.007, memory: 133.0 MB, disk 330.0 MB, gpus: 0.0, runtime 301.509766 output: Using /cvmfs certificates. 2021-10-02 22:49:05 python_package_run(160060): setting up conda environment at /var/condor/execute/dir_155377/worker-230547-155406/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:49:05 python_package_run(160060): waiting for lock on directory /var/condor/execute/dir_155377/worker-230547-155406/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:49:05 python_package_run(160060): directory /var/condor/execute/dir_155377/worker-230547-155406/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:49:05 python_package_run(160060): activating environment 2021-10-02 22:49:05 python_package_run(160060): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_155377/worker-230547-155406/t.485/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_155377/worker-230547-155406/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_155377/worker-230547-155406/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_155377/worker-230547-155406/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_155377/worker-230547-155406/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_155377/worker-230547-155406/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_155377/worker-230547-155406/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_155377/worker-230547-155406/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_155377/worker-230547-155406/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttHJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_16931.root 2021-10-02 22:54:06 python_package_run(160060): command line exit code: 1 2021-10-02 22:54:06 python_package_run(160060): total runtime: 301.410 seconds task id 485 item 353e4b0a failed: OUTPUT_MISS ('UL17_ttHJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttHJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_16931.root', 'Events', 0, 117970) 353e4b0a without result. resubmitting 353e4b0a partly as 353e4b0a. 1 attempt(s) left. processing task id 487 item 7b0b29c5 with 118118 events completed on qa-rtx6k-021.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.008, memory: 133.0 MB, disk 330.0 MB, gpus: 0.0, runtime 301.550598 output: Using /cvmfs certificates. 2021-10-02 22:49:05 python_package_run(160058): setting up conda environment at /var/condor/execute/dir_155377/worker-230547-155406/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:49:05 python_package_run(160058): waiting for lock on directory /var/condor/execute/dir_155377/worker-230547-155406/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:49:05 python_package_run(160058): directory /var/condor/execute/dir_155377/worker-230547-155406/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:49:05 python_package_run(160058): activating environment 2021-10-02 22:49:05 python_package_run(160058): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_155377/worker-230547-155406/t.487/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_155377/worker-230547-155406/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_155377/worker-230547-155406/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_155377/worker-230547-155406/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_155377/worker-230547-155406/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_155377/worker-230547-155406/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_155377/worker-230547-155406/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_155377/worker-230547-155406/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_155377/worker-230547-155406/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttHJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_16914.root 2021-10-02 22:54:06 python_package_run(160058): command line exit code: 1 2021-10-02 22:54:06 python_package_run(160058): total runtime: 301.459 seconds task id 487 item 7b0b29c5 failed: OUTPUT_MISS ('UL17_ttHJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttHJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_16914.root', 'Events', 0, 118118) 7b0b29c5 without result. resubmitting 7b0b29c5 partly as 7b0b29c5. 1 attempt(s) left. processing task id 486 item 47c87f57 with 117969 events completed on qa-rtx6k-021.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.008, memory: 133.0 MB, disk 330.0 MB, gpus: 0.0, runtime 301.594682 output: Using /cvmfs certificates. 2021-10-02 22:49:05 python_package_run(160059): setting up conda environment at /var/condor/execute/dir_155377/worker-230547-155406/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:49:05 python_package_run(160059): waiting for lock on directory /var/condor/execute/dir_155377/worker-230547-155406/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:49:05 python_package_run(160059): directory /var/condor/execute/dir_155377/worker-230547-155406/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:49:05 python_package_run(160059): activating environment 2021-10-02 22:49:05 python_package_run(160059): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_155377/worker-230547-155406/t.486/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_155377/worker-230547-155406/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_155377/worker-230547-155406/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_155377/worker-230547-155406/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_155377/worker-230547-155406/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_155377/worker-230547-155406/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_155377/worker-230547-155406/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_155377/worker-230547-155406/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_155377/worker-230547-155406/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttHJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_16931.root 2021-10-02 22:54:06 python_package_run(160059): command line exit code: 1 2021-10-02 22:54:06 python_package_run(160059): total runtime: 301.504 seconds task id 486 item 47c87f57 failed: OUTPUT_MISS ('UL17_ttHJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttHJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_16931.root', 'Events', 117970, 235939) 47c87f57 without result. resubmitting 47c87f57 partly as 47c87f57. 1 attempt(s) left. processing task id 511 item e15053f0 with 113889 events completed on qa-rtx6k-021.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.007, memory: 139.0 MB, disk 330.0 MB, gpus: 0.0, runtime 301.365112 output: Using /cvmfs certificates. 2021-10-02 22:49:11 python_package_run(160298): setting up conda environment at /var/condor/execute/dir_155377/worker-230547-155406/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:49:11 python_package_run(160298): waiting for lock on directory /var/condor/execute/dir_155377/worker-230547-155406/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:49:11 python_package_run(160298): directory /var/condor/execute/dir_155377/worker-230547-155406/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:49:11 python_package_run(160298): activating environment 2021-10-02 22:49:11 python_package_run(160298): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_155377/worker-230547-155406/t.511/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_155377/worker-230547-155406/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_155377/worker-230547-155406/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_155377/worker-230547-155406/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_155377/worker-230547-155406/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_155377/worker-230547-155406/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_155377/worker-230547-155406/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_155377/worker-230547-155406/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_155377/worker-230547-155406/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttHJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_11281.root 2021-10-02 22:54:12 python_package_run(160298): command line exit code: 1 2021-10-02 22:54:12 python_package_run(160298): total runtime: 301.274 seconds task id 511 item e15053f0 failed: OUTPUT_MISS ('UL17_ttHJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttHJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_11281.root', 'Events', 0, 113889) e15053f0 without result. resubmitting e15053f0 partly as e15053f0. 1 attempt(s) left. processing task id 402 item 5f4f7785 with 107933 events completed on qa-rtx6k-021.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.263, memory: 1141.0 MB, disk 353.0 MB, gpus: 0.0, runtime 431.897682 processing task id 512 item 28cc63de with 113888 events completed on qa-rtx6k-021.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.007, memory: 133.0 MB, disk 330.0 MB, gpus: 0.0, runtime 302.233088 output: Using /cvmfs certificates. 2021-10-02 22:49:21 python_package_run(160368): setting up conda environment at /var/condor/execute/dir_155377/worker-230547-155406/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:49:21 python_package_run(160368): waiting for lock on directory /var/condor/execute/dir_155377/worker-230547-155406/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:49:22 python_package_run(160368): directory /var/condor/execute/dir_155377/worker-230547-155406/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:49:22 python_package_run(160368): activating environment 2021-10-02 22:49:22 python_package_run(160368): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_155377/worker-230547-155406/t.512/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_155377/worker-230547-155406/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_155377/worker-230547-155406/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_155377/worker-230547-155406/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_155377/worker-230547-155406/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_155377/worker-230547-155406/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_155377/worker-230547-155406/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_155377/worker-230547-155406/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_155377/worker-230547-155406/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttHJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_11281.root 2021-10-02 22:54:24 python_package_run(160368): command line exit code: 1 2021-10-02 22:54:24 python_package_run(160368): total runtime: 302.128 seconds task id 512 item 28cc63de failed: OUTPUT_MISS ('UL17_ttHJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttHJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_11281.root', 'Events', 113889, 227777) 28cc63de without result. resubmitting 28cc63de partly as 28cc63de. 1 attempt(s) left. processing task id 509 item c6ad0ecc with 111303 events completed on d12chas551.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 134.0 MB, disk 330.0 MB, gpus: 0.0, runtime 302.891895 output: Using /cvmfs certificates. 2021-10-02 22:49:21 python_package_run(10278): setting up conda environment at /var/condor/execute/dir_7659/worker-230547-7664/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:49:21 python_package_run(10278): waiting for lock on directory /var/condor/execute/dir_7659/worker-230547-7664/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:49:22 python_package_run(10278): directory /var/condor/execute/dir_7659/worker-230547-7664/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:49:22 python_package_run(10278): activating environment 2021-10-02 22:49:22 python_package_run(10278): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_7659/worker-230547-7664/t.509/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_7659/worker-230547-7664/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_7659/worker-230547-7664/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_7659/worker-230547-7664/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_7659/worker-230547-7664/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_7659/worker-230547-7664/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_7659/worker-230547-7664/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_7659/worker-230547-7664/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_7659/worker-230547-7664/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttHJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_12012.root 2021-10-02 22:54:24 python_package_run(10278): command line exit code: 1 2021-10-02 22:54:24 python_package_run(10278): total runtime: 302.766 seconds task id 509 item c6ad0ecc failed: OUTPUT_MISS ('UL17_ttHJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttHJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_12012.root', 'Events', 0, 111303) c6ad0ecc without result. resubmitting c6ad0ecc partly as c6ad0ecc. 1 attempt(s) left. processing task id 520 item 84c63233 with 84000 events completed on d12chas580.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 138.0 MB, disk 330.0 MB, gpus: 0.0, runtime 291.936689 output: Using /cvmfs certificates. 2021-10-02 22:49:33 python_package_run(9826): setting up conda environment at /var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:49:33 python_package_run(9826): waiting for lock on directory /var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:49:33 python_package_run(9826): directory /var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:49:33 python_package_run(9826): activating environment 2021-10-02 22:49:33 python_package_run(9826): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_5228/worker-230547-5303/t.520/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_tllq4fNoSchanWNoHiggs0p_all22WCsStartPtCheckV2dim6TopMay20GST_run0/NAOD-00000_7991.root 2021-10-02 22:54:25 python_package_run(9826): command line exit code: 1 2021-10-02 22:54:25 python_package_run(9826): total runtime: 291.802 seconds task id 520 item 84c63233 failed: OUTPUT_MISS ('UL17_tllq_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_tllq4fNoSchanWNoHiggs0p_all22WCsStartPtCheckV2dim6TopMay20GST_run0/NAOD-00000_7991.root', 'Events', 0, 84000) 84c63233 without result. resubmitting 84c63233 partly as 84c63233. 1 attempt(s) left. processing task id 508 item 108cb9b2 with 112585 events completed on qa-rtx6k-021.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.007, memory: 133.0 MB, disk 330.0 MB, gpus: 0.0, runtime 302.304323 output: Using /cvmfs certificates. 2021-10-02 22:49:25 python_package_run(160432): setting up conda environment at /var/condor/execute/dir_155377/worker-230547-155406/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:49:25 python_package_run(160432): waiting for lock on directory /var/condor/execute/dir_155377/worker-230547-155406/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:49:26 python_package_run(160432): directory /var/condor/execute/dir_155377/worker-230547-155406/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:49:26 python_package_run(160432): activating environment 2021-10-02 22:49:26 python_package_run(160432): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_155377/worker-230547-155406/t.508/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_155377/worker-230547-155406/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_155377/worker-230547-155406/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_155377/worker-230547-155406/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_155377/worker-230547-155406/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_155377/worker-230547-155406/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_155377/worker-230547-155406/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_155377/worker-230547-155406/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_155377/worker-230547-155406/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttHJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_12743.root 2021-10-02 22:54:28 python_package_run(160432): command line exit code: 1 2021-10-02 22:54:28 python_package_run(160432): total runtime: 302.197 seconds task id 508 item 108cb9b2 failed: OUTPUT_MISS ('UL17_ttHJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttHJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_12743.root', 'Events', 112585, 225170) 108cb9b2 without result. resubmitting 108cb9b2 partly as 108cb9b2. 1 attempt(s) left. processing task id 513 item 61d7ae8d with 107507 events completed on d12chas547.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 133.0 MB, disk 330.0 MB, gpus: 0.0, runtime 302.833399 output: Using /cvmfs certificates. 2021-10-02 22:49:25 python_package_run(4155): setting up conda environment at /var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:49:25 python_package_run(4155): waiting for lock on directory /var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:49:25 python_package_run(4155): directory /var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:49:25 python_package_run(4155): activating environment 2021-10-02 22:49:26 python_package_run(4155): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_8819/worker-230547-8824/t.513/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttHJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_10199.root 2021-10-02 22:54:28 python_package_run(4155): command line exit code: 1 2021-10-02 22:54:28 python_package_run(4155): total runtime: 302.751 seconds task id 513 item 61d7ae8d failed: OUTPUT_MISS ('UL17_ttHJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttHJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_10199.root', 'Events', 0, 107507) 61d7ae8d without result. resubmitting 61d7ae8d partly as 61d7ae8d. 1 attempt(s) left. processing task id 514 item c596506b with 107506 events completed on d12chas548.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 136.0 MB, disk 330.0 MB, gpus: 0.0, runtime 302.930908 output: Using /cvmfs certificates. 2021-10-02 22:49:25 python_package_run(26000): setting up conda environment at /var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:49:25 python_package_run(26000): waiting for lock on directory /var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:49:26 python_package_run(26000): directory /var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:49:26 python_package_run(26000): activating environment 2021-10-02 22:49:26 python_package_run(26000): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_24697/worker-230547-24702/t.514/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttHJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_10199.root 2021-10-02 22:54:28 python_package_run(26000): command line exit code: 1 2021-10-02 22:54:28 python_package_run(26000): total runtime: 302.812 seconds task id 514 item c596506b failed: OUTPUT_MISS ('UL17_ttHJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttHJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_10199.root', 'Events', 107507, 215013) c596506b without result. resubmitting c596506b partly as c596506b. 1 attempt(s) left. processing task id 516 item e09a20c8 with 84167 events completed on d12chas548.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 138.0 MB, disk 330.0 MB, gpus: 0.0, runtime 302.93132 output: Using /cvmfs certificates. 2021-10-02 22:49:26 python_package_run(26054): setting up conda environment at /var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:49:26 python_package_run(26054): waiting for lock on directory /var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:49:26 python_package_run(26054): directory /var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:49:26 python_package_run(26054): activating environment 2021-10-02 22:49:27 python_package_run(26054): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_24697/worker-230547-24702/t.516/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_24697/worker-230547-24702/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_tllq4fNoSchanWNoHiggs0p_all22WCsStartPtCheckV2dim6TopMay20GST_run0/NAOD-00000_9765.root 2021-10-02 22:54:29 python_package_run(26054): command line exit code: 1 2021-10-02 22:54:29 python_package_run(26054): total runtime: 302.823 seconds task id 516 item e09a20c8 failed: OUTPUT_MISS ('UL17_tllq_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_tllq4fNoSchanWNoHiggs0p_all22WCsStartPtCheckV2dim6TopMay20GST_run0/NAOD-00000_9765.root', 'Events', 84167, 168334) e09a20c8 without result. resubmitting e09a20c8 partly as e09a20c8. 1 attempt(s) left. processing task id 519 item bfac42b9 with 122000 events completed on d12chas553.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.012, memory: 133.0 MB, disk 330.0 MB, gpus: 0.0, runtime 303.673609 output: Using /cvmfs certificates. 2021-10-02 22:49:30 python_package_run(16412): setting up conda environment at /var/condor/execute/dir_12055/worker-230547-12064/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:49:30 python_package_run(16412): waiting for lock on directory /var/condor/execute/dir_12055/worker-230547-12064/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:49:30 python_package_run(16412): directory /var/condor/execute/dir_12055/worker-230547-12064/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:49:30 python_package_run(16412): activating environment 2021-10-02 22:49:30 python_package_run(16412): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_12055/worker-230547-12064/t.519/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_12055/worker-230547-12064/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_12055/worker-230547-12064/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_12055/worker-230547-12064/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_12055/worker-230547-12064/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_12055/worker-230547-12064/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_12055/worker-230547-12064/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_12055/worker-230547-12064/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_12055/worker-230547-12064/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_tllq4fNoSchanWNoHiggs0p_all22WCsStartPtCheckV2dim6TopMay20GST_run0/NAOD-00000_8727.root 2021-10-02 22:54:33 python_package_run(16412): command line exit code: 1 2021-10-02 22:54:33 python_package_run(16412): total runtime: 303.554 seconds task id 519 item bfac42b9 failed: OUTPUT_MISS ('UL17_tllq_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_tllq4fNoSchanWNoHiggs0p_all22WCsStartPtCheckV2dim6TopMay20GST_run0/NAOD-00000_8727.root', 'Events', 122000, 244000) bfac42b9 without result. resubmitting bfac42b9 partly as bfac42b9. 1 attempt(s) left. processing task id 405 item fef67bc0 with 112931 events completed on d12chas589.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.358, memory: 1227.0 MB, disk 353.0 MB, gpus: 0.0, runtime 446.978381 submitted processing task id 700 item e742823e, with 84000 events processing task id 521 item ce1334d4 with 84000 events completed on d12chas546.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 134.0 MB, disk 330.0 MB, gpus: 0.0, runtime 299.439765 output: Using /cvmfs certificates. 2021-10-02 22:49:35 python_package_run(28860): setting up conda environment at /var/condor/execute/dir_24407/worker-230547-24421/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:49:35 python_package_run(28860): waiting for lock on directory /var/condor/execute/dir_24407/worker-230547-24421/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:49:35 python_package_run(28860): directory /var/condor/execute/dir_24407/worker-230547-24421/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:49:35 python_package_run(28860): activating environment 2021-10-02 22:49:35 python_package_run(28860): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_24407/worker-230547-24421/t.521/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_24407/worker-230547-24421/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_24407/worker-230547-24421/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_24407/worker-230547-24421/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_24407/worker-230547-24421/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_24407/worker-230547-24421/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_24407/worker-230547-24421/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_24407/worker-230547-24421/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_24407/worker-230547-24421/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_tllq4fNoSchanWNoHiggs0p_all22WCsStartPtCheckV2dim6TopMay20GST_run0/NAOD-00000_7991.root 2021-10-02 22:54:34 python_package_run(28860): command line exit code: 1 2021-10-02 22:54:34 python_package_run(28860): total runtime: 299.294 seconds task id 521 item ce1334d4 failed: OUTPUT_MISS ('UL17_tllq_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_tllq4fNoSchanWNoHiggs0p_all22WCsStartPtCheckV2dim6TopMay20GST_run0/NAOD-00000_7991.root', 'Events', 84000, 168000) ce1334d4 without result. resubmitting ce1334d4 partly as ce1334d4. 1 attempt(s) left. processing task id 342 item 29c9b6ca with 94500 events completed on d12chas546.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.358, memory: 1112.0 MB, disk 354.0 MB, gpus: 0.0, runtime 466.899469 submitted processing task id 702 item 533a94de, with 84000 events processing task id 351 item 2c231605 with 94500 events completed on d12chas547.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.337, memory: 1111.0 MB, disk 354.0 MB, gpus: 0.0, runtime 466.653171 submitted processing task id 703 item a734f430, with 15000 events processing task id 349 item 7929f3f3 with 94500 events completed on d12chas552.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.368, memory: 1160.0 MB, disk 355.0 MB, gpus: 0.0, runtime 464.283269 submitted processing task id 704 item 2384d4e3, with 115750 events processing task id 522 item ed0f0f1d with 84000 events completed on d12chas589.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 141.0 MB, disk 330.0 MB, gpus: 0.0, runtime 303.033893 output: Using /cvmfs certificates. 2021-10-02 22:49:36 python_package_run(20634): setting up conda environment at /var/condor/execute/dir_17507/worker-230547-17512/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:49:36 python_package_run(20634): waiting for lock on directory /var/condor/execute/dir_17507/worker-230547-17512/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:49:36 python_package_run(20634): directory /var/condor/execute/dir_17507/worker-230547-17512/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:49:36 python_package_run(20634): activating environment 2021-10-02 22:49:36 python_package_run(20634): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_17507/worker-230547-17512/t.522/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_17507/worker-230547-17512/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_17507/worker-230547-17512/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_17507/worker-230547-17512/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_17507/worker-230547-17512/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_17507/worker-230547-17512/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_17507/worker-230547-17512/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_17507/worker-230547-17512/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_17507/worker-230547-17512/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_tllq4fNoSchanWNoHiggs0p_all22WCsStartPtCheckV2dim6TopMay20GST_run0/NAOD-00000_7991.root 2021-10-02 22:54:39 python_package_run(20634): command line exit code: 1 2021-10-02 22:54:39 python_package_run(20634): total runtime: 302.898 seconds task id 522 item ed0f0f1d failed: OUTPUT_MISS ('UL17_tllq_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_tllq4fNoSchanWNoHiggs0p_all22WCsStartPtCheckV2dim6TopMay20GST_run0/NAOD-00000_7991.root', 'Events', 168000, 252000) ed0f0f1d without result. resubmitting ed0f0f1d partly as ed0f0f1d. 1 attempt(s) left. processing task id 523 item f5d0969e with 114750 events completed on d12chas553.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.012, memory: 133.0 MB, disk 330.0 MB, gpus: 0.0, runtime 303.684439 output: Using /cvmfs certificates. 2021-10-02 22:49:38 python_package_run(16479): setting up conda environment at /var/condor/execute/dir_12055/worker-230547-12064/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:49:38 python_package_run(16479): waiting for lock on directory /var/condor/execute/dir_12055/worker-230547-12064/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:49:38 python_package_run(16479): directory /var/condor/execute/dir_12055/worker-230547-12064/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:49:38 python_package_run(16479): activating environment 2021-10-02 22:49:38 python_package_run(16479): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_12055/worker-230547-12064/t.523/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_12055/worker-230547-12064/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_12055/worker-230547-12064/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_12055/worker-230547-12064/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_12055/worker-230547-12064/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_12055/worker-230547-12064/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_12055/worker-230547-12064/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_12055/worker-230547-12064/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_12055/worker-230547-12064/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_tllq4fNoSchanWNoHiggs0p_all22WCsStartPtCheckV2dim6TopMay20GST_run0/NAOD-00000_7294.root 2021-10-02 22:54:41 python_package_run(16479): command line exit code: 1 2021-10-02 22:54:41 python_package_run(16479): total runtime: 303.562 seconds task id 523 item f5d0969e failed: OUTPUT_MISS ('UL17_tllq_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_tllq4fNoSchanWNoHiggs0p_all22WCsStartPtCheckV2dim6TopMay20GST_run0/NAOD-00000_7294.root', 'Events', 0, 114750) f5d0969e without result. resubmitting f5d0969e partly as f5d0969e. 1 attempt(s) left. processing task id 361 item 99be0d67 with 113392 events completed on d12chas587.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.353, memory: 1204.0 MB, disk 352.0 MB, gpus: 0.0, runtime 469.25064 submitted processing task id 707 item 2933c61b, with 115750 events processing task id 524 item 96480b02 with 114750 events completed on d12chas553.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 133.0 MB, disk 330.0 MB, gpus: 0.0, runtime 303.190343 output: Using /cvmfs certificates. 2021-10-02 22:49:40 python_package_run(16535): setting up conda environment at /var/condor/execute/dir_12055/worker-230547-12064/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:49:40 python_package_run(16535): waiting for lock on directory /var/condor/execute/dir_12055/worker-230547-12064/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:49:40 python_package_run(16535): directory /var/condor/execute/dir_12055/worker-230547-12064/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:49:40 python_package_run(16535): activating environment 2021-10-02 22:49:40 python_package_run(16535): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_12055/worker-230547-12064/t.524/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_12055/worker-230547-12064/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_12055/worker-230547-12064/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_12055/worker-230547-12064/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_12055/worker-230547-12064/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_12055/worker-230547-12064/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_12055/worker-230547-12064/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_12055/worker-230547-12064/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_12055/worker-230547-12064/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_tllq4fNoSchanWNoHiggs0p_all22WCsStartPtCheckV2dim6TopMay20GST_run0/NAOD-00000_7294.root 2021-10-02 22:54:43 python_package_run(16535): command line exit code: 1 2021-10-02 22:54:43 python_package_run(16535): total runtime: 303.073 seconds task id 524 item 96480b02 failed: OUTPUT_MISS ('UL17_tllq_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_tllq4fNoSchanWNoHiggs0p_all22WCsStartPtCheckV2dim6TopMay20GST_run0/NAOD-00000_7294.root', 'Events', 114750, 229500) 96480b02 without result. resubmitting 96480b02 partly as 96480b02. 1 attempt(s) left. processing task id 525 item fb4260f6 with 123250 events completed on d12chas553.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 134.0 MB, disk 330.0 MB, gpus: 0.0, runtime 302.800191 output: Using /cvmfs certificates. 2021-10-02 22:49:40 python_package_run(16589): setting up conda environment at /var/condor/execute/dir_12055/worker-230547-12064/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:49:40 python_package_run(16589): waiting for lock on directory /var/condor/execute/dir_12055/worker-230547-12064/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:49:40 python_package_run(16589): directory /var/condor/execute/dir_12055/worker-230547-12064/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:49:40 python_package_run(16589): activating environment 2021-10-02 22:49:41 python_package_run(16589): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_12055/worker-230547-12064/t.525/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_12055/worker-230547-12064/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_12055/worker-230547-12064/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_12055/worker-230547-12064/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_12055/worker-230547-12064/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_12055/worker-230547-12064/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_12055/worker-230547-12064/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_12055/worker-230547-12064/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_12055/worker-230547-12064/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_tllq4fNoSchanWNoHiggs0p_all22WCsStartPtCheckV2dim6TopMay20GST_run0/NAOD-00000_6575.root 2021-10-02 22:54:43 python_package_run(16589): command line exit code: 1 2021-10-02 22:54:43 python_package_run(16589): total runtime: 302.687 seconds task id 525 item fb4260f6 failed: OUTPUT_MISS ('UL17_tllq_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_tllq4fNoSchanWNoHiggs0p_all22WCsStartPtCheckV2dim6TopMay20GST_run0/NAOD-00000_6575.root', 'Events', 0, 123250) fb4260f6 without result. resubmitting fb4260f6 partly as fb4260f6. 1 attempt(s) left. processing task id 526 item 5e7035ac with 123250 events completed on d12chas553.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.009, memory: 134.0 MB, disk 330.0 MB, gpus: 0.0, runtime 302.802297 output: Using /cvmfs certificates. 2021-10-02 22:49:41 python_package_run(16649): setting up conda environment at /var/condor/execute/dir_12056/worker-230547-12074/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:49:41 python_package_run(16649): waiting for lock on directory /var/condor/execute/dir_12056/worker-230547-12074/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:49:41 python_package_run(16649): directory /var/condor/execute/dir_12056/worker-230547-12074/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:49:41 python_package_run(16649): activating environment 2021-10-02 22:49:41 python_package_run(16649): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_12056/worker-230547-12074/t.526/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_12056/worker-230547-12074/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_12056/worker-230547-12074/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_12056/worker-230547-12074/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_12056/worker-230547-12074/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_12056/worker-230547-12074/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_12056/worker-230547-12074/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_12056/worker-230547-12074/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_12056/worker-230547-12074/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_tllq4fNoSchanWNoHiggs0p_all22WCsStartPtCheckV2dim6TopMay20GST_run0/NAOD-00000_6575.root 2021-10-02 22:54:44 python_package_run(16649): command line exit code: 1 2021-10-02 22:54:44 python_package_run(16649): total runtime: 302.682 seconds task id 526 item 5e7035ac failed: OUTPUT_MISS ('UL17_tllq_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_tllq4fNoSchanWNoHiggs0p_all22WCsStartPtCheckV2dim6TopMay20GST_run0/NAOD-00000_6575.root', 'Events', 123250, 246500) 5e7035ac without result. resubmitting 5e7035ac partly as 5e7035ac. 1 attempt(s) left. processing task id 528 item 1fdd31e5 with 84334 events completed on d12chas564.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 134.0 MB, disk 330.0 MB, gpus: 0.0, runtime 302.813115 output: Using /cvmfs certificates. 2021-10-02 22:49:43 python_package_run(26976): setting up conda environment at /var/condor/execute/dir_23235/worker-230547-23244/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:49:43 python_package_run(26976): waiting for lock on directory /var/condor/execute/dir_23235/worker-230547-23244/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:49:43 python_package_run(26976): directory /var/condor/execute/dir_23235/worker-230547-23244/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:49:43 python_package_run(26976): activating environment 2021-10-02 22:49:43 python_package_run(26976): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_23235/worker-230547-23244/t.528/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_23235/worker-230547-23244/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_23235/worker-230547-23244/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_23235/worker-230547-23244/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_23235/worker-230547-23244/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_23235/worker-230547-23244/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_23235/worker-230547-23244/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_23235/worker-230547-23244/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_23235/worker-230547-23244/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_tllq4fNoSchanWNoHiggs0p_all22WCsStartPtCheckV2dim6TopMay20GST_run0/NAOD-00000_5897.root 2021-10-02 22:54:46 python_package_run(26976): command line exit code: 1 2021-10-02 22:54:46 python_package_run(26976): total runtime: 302.677 seconds task id 528 item 1fdd31e5 failed: OUTPUT_MISS ('UL17_tllq_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_tllq4fNoSchanWNoHiggs0p_all22WCsStartPtCheckV2dim6TopMay20GST_run0/NAOD-00000_5897.root', 'Events', 84334, 168668) 1fdd31e5 without result. resubmitting 1fdd31e5 partly as 1fdd31e5. 1 attempt(s) left. submitted processing task id 712 item 5ea15449, with 116500 events processing task id 529 item eed89c36 with 84332 events completed on d12chas553.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 134.0 MB, disk 330.0 MB, gpus: 0.0, runtime 302.965297 output: Using /cvmfs certificates. 2021-10-02 22:49:48 python_package_run(16732): setting up conda environment at /var/condor/execute/dir_12055/worker-230547-12064/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:49:48 python_package_run(16732): waiting for lock on directory /var/condor/execute/dir_12055/worker-230547-12064/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:49:48 python_package_run(16732): directory /var/condor/execute/dir_12055/worker-230547-12064/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:49:48 python_package_run(16732): activating environment 2021-10-02 22:49:48 python_package_run(16732): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_12055/worker-230547-12064/t.529/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_12055/worker-230547-12064/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_12055/worker-230547-12064/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_12055/worker-230547-12064/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_12055/worker-230547-12064/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_12055/worker-230547-12064/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_12055/worker-230547-12064/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_12055/worker-230547-12064/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_12055/worker-230547-12064/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_tllq4fNoSchanWNoHiggs0p_all22WCsStartPtCheckV2dim6TopMay20GST_run0/NAOD-00000_5897.root 2021-10-02 22:54:51 python_package_run(16732): command line exit code: 1 2021-10-02 22:54:51 python_package_run(16732): total runtime: 302.830 seconds task id 529 item eed89c36 failed: OUTPUT_MISS ('UL17_tllq_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_tllq4fNoSchanWNoHiggs0p_all22WCsStartPtCheckV2dim6TopMay20GST_run0/NAOD-00000_5897.root', 'Events', 168668, 253000) eed89c36 without result. resubmitting eed89c36 partly as eed89c36. 1 attempt(s) left. processing task id 530 item 3c0912a7 with 117250 events completed on d12chas551.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 133.0 MB, disk 330.0 MB, gpus: 0.0, runtime 302.727591 output: Using /cvmfs certificates. 2021-10-02 22:50:01 python_package_run(10478): setting up conda environment at /var/condor/execute/dir_7659/worker-230547-7664/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:50:01 python_package_run(10478): waiting for lock on directory /var/condor/execute/dir_7659/worker-230547-7664/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:50:01 python_package_run(10478): directory /var/condor/execute/dir_7659/worker-230547-7664/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:50:01 python_package_run(10478): activating environment 2021-10-02 22:50:01 python_package_run(10478): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_7659/worker-230547-7664/t.530/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_7659/worker-230547-7664/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_7659/worker-230547-7664/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_7659/worker-230547-7664/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_7659/worker-230547-7664/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_7659/worker-230547-7664/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_7659/worker-230547-7664/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_7659/worker-230547-7664/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_7659/worker-230547-7664/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_tllq4fNoSchanWNoHiggs0p_all22WCsStartPtCheckV2dim6TopMay20GST_run0/NAOD-00000_5567.root 2021-10-02 22:55:03 python_package_run(10478): command line exit code: 1 2021-10-02 22:55:03 python_package_run(10478): total runtime: 302.595 seconds task id 530 item 3c0912a7 failed: OUTPUT_MISS ('UL17_tllq_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_tllq4fNoSchanWNoHiggs0p_all22WCsStartPtCheckV2dim6TopMay20GST_run0/NAOD-00000_5567.root', 'Events', 0, 117250) 3c0912a7 without result. resubmitting 3c0912a7 partly as 3c0912a7. 1 attempt(s) left. processing task id 531 item 9b6c849e with 117250 events completed on d12chas580.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 140.0 MB, disk 330.0 MB, gpus: 0.0, runtime 303.108068 output: Using /cvmfs certificates. 2021-10-02 22:50:02 python_package_run(9921): setting up conda environment at /var/condor/execute/dir_5229/worker-230547-5310/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:50:02 python_package_run(9921): waiting for lock on directory /var/condor/execute/dir_5229/worker-230547-5310/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:50:03 python_package_run(9921): directory /var/condor/execute/dir_5229/worker-230547-5310/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:50:03 python_package_run(9921): activating environment 2021-10-02 22:50:03 python_package_run(9921): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_5229/worker-230547-5310/t.531/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_5229/worker-230547-5310/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_5229/worker-230547-5310/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_5229/worker-230547-5310/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_5229/worker-230547-5310/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_5229/worker-230547-5310/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_5229/worker-230547-5310/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_5229/worker-230547-5310/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_5229/worker-230547-5310/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_tllq4fNoSchanWNoHiggs0p_all22WCsStartPtCheckV2dim6TopMay20GST_run0/NAOD-00000_5567.root 2021-10-02 22:55:05 python_package_run(9921): command line exit code: 1 2021-10-02 22:55:05 python_package_run(9921): total runtime: 302.977 seconds task id 531 item 9b6c849e failed: OUTPUT_MISS ('UL17_tllq_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_tllq4fNoSchanWNoHiggs0p_all22WCsStartPtCheckV2dim6TopMay20GST_run0/NAOD-00000_5567.root', 'Events', 117250, 234500) 9b6c849e without result. resubmitting 9b6c849e partly as 9b6c849e. 1 attempt(s) left. processing task id 532 item e44ecb58 with 84000 events completed on d12chas547.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 134.0 MB, disk 330.0 MB, gpus: 0.0, runtime 302.931609 output: Using /cvmfs certificates. 2021-10-02 22:50:11 python_package_run(4490): setting up conda environment at /var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:50:11 python_package_run(4490): waiting for lock on directory /var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:50:11 python_package_run(4490): directory /var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:50:11 python_package_run(4490): activating environment 2021-10-02 22:50:11 python_package_run(4490): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_8819/worker-230547-8824/t.532/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_tllq4fNoSchanWNoHiggs0p_all22WCsStartPtCheckV2dim6TopMay20GST_run0/NAOD-00000_4783.root 2021-10-02 22:55:13 python_package_run(4490): command line exit code: 1 2021-10-02 22:55:13 python_package_run(4490): total runtime: 302.815 seconds task id 532 item e44ecb58 failed: OUTPUT_MISS ('UL17_tllq_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_tllq4fNoSchanWNoHiggs0p_all22WCsStartPtCheckV2dim6TopMay20GST_run0/NAOD-00000_4783.root', 'Events', 0, 84000) e44ecb58 without result. resubmitting e44ecb58 partly as e44ecb58. 1 attempt(s) left. processing task id 534 item bee9251a with 84000 events completed on d12chas547.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.009, memory: 134.0 MB, disk 330.0 MB, gpus: 0.0, runtime 302.659897 output: Using /cvmfs certificates. 2021-10-02 22:50:29 python_package_run(4684): setting up conda environment at /var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:50:29 python_package_run(4684): waiting for lock on directory /var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:50:29 python_package_run(4684): directory /var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:50:29 python_package_run(4684): activating environment 2021-10-02 22:50:29 python_package_run(4684): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_8819/worker-230547-8824/t.534/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_tllq4fNoSchanWNoHiggs0p_all22WCsStartPtCheckV2dim6TopMay20GST_run0/NAOD-00000_4783.root 2021-10-02 22:55:32 python_package_run(4684): command line exit code: 1 2021-10-02 22:55:32 python_package_run(4684): total runtime: 302.549 seconds task id 534 item bee9251a failed: OUTPUT_MISS ('UL17_tllq_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_tllq4fNoSchanWNoHiggs0p_all22WCsStartPtCheckV2dim6TopMay20GST_run0/NAOD-00000_4783.root', 'Events', 84000, 168000) bee9251a without result. resubmitting bee9251a partly as bee9251a. 1 attempt(s) left. processing task id 543 item 34a284aa with 94500 events completed on d12chas547.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 137.0 MB, disk 330.0 MB, gpus: 0.0, runtime 292.792159 output: Using /cvmfs certificates. 2021-10-02 22:51:54 python_package_run(5116): setting up conda environment at /var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:51:54 python_package_run(5116): waiting for lock on directory /var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:51:54 python_package_run(5116): directory /var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:51:54 python_package_run(5116): activating environment 2021-10-02 22:51:54 python_package_run(5116): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_8819/worker-230547-8824/t.543/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_10088.root 2021-10-02 22:56:47 python_package_run(5116): command line exit code: 1 2021-10-02 22:56:47 python_package_run(5116): total runtime: 292.678 seconds task id 543 item 34a284aa failed: OUTPUT_MISS ('UL17_tttt_b4', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_10088.root', 'Events', 94500, 189000) 34a284aa without result. resubmitting 34a284aa partly as 34a284aa. 0 attempt(s) left. processing task id 547 item 056212f9 with 94500 events completed on d12chas580.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 134.0 MB, disk 330.0 MB, gpus: 0.0, runtime 292.581641 output: Using /cvmfs certificates. 2021-10-02 22:51:54 python_package_run(10722): setting up conda environment at /var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:51:54 python_package_run(10722): waiting for lock on directory /var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:51:54 python_package_run(10722): directory /var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:51:54 python_package_run(10722): activating environment 2021-10-02 22:51:54 python_package_run(10722): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_5228/worker-230547-5303/t.547/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_10087.root 2021-10-02 22:56:47 python_package_run(10722): command line exit code: 1 2021-10-02 22:56:47 python_package_run(10722): total runtime: 292.496 seconds task id 547 item 056212f9 failed: OUTPUT_MISS ('UL17_tttt_b4', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_10087.root', 'Events', 0, 94500) 056212f9 without result. resubmitting 056212f9 partly as 056212f9. 0 attempt(s) left. processing task id 545 item e5241d59 with 94500 events completed on d12chas580.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 138.0 MB, disk 330.0 MB, gpus: 0.0, runtime 292.851111 output: Using /cvmfs certificates. 2021-10-02 22:51:54 python_package_run(10598): setting up conda environment at /var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:51:54 python_package_run(10598): waiting for lock on directory /var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:51:54 python_package_run(10598): directory /var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:51:54 python_package_run(10598): activating environment 2021-10-02 22:51:54 python_package_run(10598): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_5228/worker-230547-5303/t.545/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_10087.root 2021-10-02 22:56:47 python_package_run(10598): command line exit code: 1 2021-10-02 22:56:47 python_package_run(10598): total runtime: 292.768 seconds task id 545 item e5241d59 failed: OUTPUT_MISS ('UL17_tttt_b4', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_10087.root', 'Events', 94500, 189000) e5241d59 without result. resubmitting e5241d59 partly as e5241d59. 0 attempt(s) left. processing task id 546 item 01a58bfd with 94500 events completed on d12chas580.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 140.0 MB, disk 330.0 MB, gpus: 0.0, runtime 292.658834 output: Using /cvmfs certificates. 2021-10-02 22:51:54 python_package_run(10696): setting up conda environment at /var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:51:54 python_package_run(10696): waiting for lock on directory /var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:51:54 python_package_run(10696): directory /var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:51:54 python_package_run(10696): activating environment 2021-10-02 22:51:54 python_package_run(10696): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_5228/worker-230547-5303/t.546/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_10086.root 2021-10-02 22:56:47 python_package_run(10696): command line exit code: 1 2021-10-02 22:56:47 python_package_run(10696): total runtime: 292.579 seconds task id 546 item 01a58bfd failed: OUTPUT_MISS ('UL17_tttt_b4', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_10086.root', 'Events', 94500, 189000) 01a58bfd without result. resubmitting 01a58bfd partly as 01a58bfd. 0 attempt(s) left. processing task id 542 item 7d52b268 with 94500 events completed on d12chas564.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 133.0 MB, disk 330.0 MB, gpus: 0.0, runtime 294.82596 output: Using /cvmfs certificates. 2021-10-02 22:51:52 python_package_run(27368): setting up conda environment at /var/condor/execute/dir_23236/worker-230547-23258/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:51:52 python_package_run(27368): waiting for lock on directory /var/condor/execute/dir_23236/worker-230547-23258/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:51:52 python_package_run(27368): directory /var/condor/execute/dir_23236/worker-230547-23258/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:51:52 python_package_run(27368): activating environment 2021-10-02 22:51:52 python_package_run(27368): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_23236/worker-230547-23258/t.542/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_23236/worker-230547-23258/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_23236/worker-230547-23258/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_23236/worker-230547-23258/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_23236/worker-230547-23258/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_23236/worker-230547-23258/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_23236/worker-230547-23258/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_23236/worker-230547-23258/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_23236/worker-230547-23258/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_10093.root 2021-10-02 22:56:47 python_package_run(27368): command line exit code: 1 2021-10-02 22:56:47 python_package_run(27368): total runtime: 294.689 seconds task id 542 item 7d52b268 failed: OUTPUT_MISS ('UL17_tttt_b4', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_10093.root', 'Events', 0, 94500) 7d52b268 without result. resubmitting 7d52b268 partly as 7d52b268. 0 attempt(s) left. processing task id 539 item d4f5ccfc with 94500 events completed on d12chas580.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 133.0 MB, disk 330.0 MB, gpus: 0.0, runtime 295.396468 output: Using /cvmfs certificates. 2021-10-02 22:51:52 python_package_run(10339): setting up conda environment at /var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:51:52 python_package_run(10339): waiting for lock on directory /var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:51:52 python_package_run(10339): directory /var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:51:52 python_package_run(10339): activating environment 2021-10-02 22:51:52 python_package_run(10339): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_5228/worker-230547-5303/t.539/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_10092.root 2021-10-02 22:56:47 python_package_run(10339): command line exit code: 1 2021-10-02 22:56:47 python_package_run(10339): total runtime: 295.310 seconds task id 539 item d4f5ccfc failed: OUTPUT_MISS ('UL17_tttt_b4', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_10092.root', 'Events', 94500, 189000) d4f5ccfc without result. resubmitting d4f5ccfc partly as d4f5ccfc. 0 attempt(s) left. processing task id 537 item 5fcba9fa with 94250 events completed on d12chas580.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 140.0 MB, disk 330.0 MB, gpus: 0.0, runtime 295.559709 output: Using /cvmfs certificates. 2021-10-02 22:51:51 python_package_run(10288): setting up conda environment at /var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:51:51 python_package_run(10288): waiting for lock on directory /var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:51:52 python_package_run(10288): directory /var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:51:52 python_package_run(10288): activating environment 2021-10-02 22:51:52 python_package_run(10288): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_5228/worker-230547-5303/t.537/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_10090.root 2021-10-02 22:56:47 python_package_run(10288): command line exit code: 1 2021-10-02 22:56:47 python_package_run(10288): total runtime: 295.446 seconds task id 537 item 5fcba9fa failed: OUTPUT_MISS ('UL17_tttt_b4', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_10090.root', 'Events', 94250, 188500) 5fcba9fa without result. resubmitting 5fcba9fa partly as 5fcba9fa. 0 attempt(s) left. processing task id 540 item 27aaaee0 with 94500 events completed on d12chas580.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.012, memory: 135.0 MB, disk 330.0 MB, gpus: 0.0, runtime 295.464569 output: Using /cvmfs certificates. 2021-10-02 22:51:52 python_package_run(10397): setting up conda environment at /var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:51:52 python_package_run(10397): waiting for lock on directory /var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:51:52 python_package_run(10397): directory /var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:51:52 python_package_run(10397): activating environment 2021-10-02 22:51:52 python_package_run(10397): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_5228/worker-230547-5303/t.540/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_10091.root 2021-10-02 22:56:47 python_package_run(10397): command line exit code: 1 2021-10-02 22:56:47 python_package_run(10397): total runtime: 295.379 seconds task id 540 item 27aaaee0 failed: OUTPUT_MISS ('UL17_tttt_b4', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_10091.root', 'Events', 94500, 189000) 27aaaee0 without result. resubmitting 27aaaee0 partly as 27aaaee0. 0 attempt(s) left. processing task id 538 item 3e008e90 with 94250 events completed on d12chas580.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 137.0 MB, disk 330.0 MB, gpus: 0.0, runtime 295.599796 output: Using /cvmfs certificates. 2021-10-02 22:51:52 python_package_run(10309): setting up conda environment at /var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:51:52 python_package_run(10309): waiting for lock on directory /var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:51:52 python_package_run(10309): directory /var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:51:52 python_package_run(10309): activating environment 2021-10-02 22:51:52 python_package_run(10309): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_5228/worker-230547-5303/t.538/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_10090.root 2021-10-02 22:56:47 python_package_run(10309): command line exit code: 1 2021-10-02 22:56:47 python_package_run(10309): total runtime: 295.497 seconds task id 538 item 3e008e90 failed: OUTPUT_MISS ('UL17_tttt_b4', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_10090.root', 'Events', 0, 94250) 3e008e90 without result. resubmitting 3e008e90 partly as 3e008e90. 0 attempt(s) left. processing task id 548 item 43010f60 with 94500 events completed on d12chas547.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 133.0 MB, disk 330.0 MB, gpus: 0.0, runtime 292.864829 output: Using /cvmfs certificates. 2021-10-02 22:51:54 python_package_run(5169): setting up conda environment at /var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:51:54 python_package_run(5169): waiting for lock on directory /var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:51:54 python_package_run(5169): directory /var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:51:54 python_package_run(5169): activating environment 2021-10-02 22:51:54 python_package_run(5169): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_8819/worker-230547-8824/t.548/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_8819/worker-230547-8824/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_10089.root 2021-10-02 22:56:47 python_package_run(5169): command line exit code: 1 2021-10-02 22:56:47 python_package_run(5169): total runtime: 292.754 seconds task id 548 item 43010f60 failed: OUTPUT_MISS ('UL17_tttt_b4', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_10089.root', 'Events', 94500, 189000) 43010f60 without result. resubmitting 43010f60 partly as 43010f60. 0 attempt(s) left. processing task id 544 item 0e2e8207 with 94500 events completed on d12chas580.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.009, memory: 137.0 MB, disk 330.0 MB, gpus: 0.0, runtime 293.3194 output: Using /cvmfs certificates. 2021-10-02 22:51:54 python_package_run(10569): setting up conda environment at /var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:51:54 python_package_run(10569): waiting for lock on directory /var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:51:54 python_package_run(10569): directory /var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:51:54 python_package_run(10569): activating environment 2021-10-02 22:51:54 python_package_run(10569): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_5228/worker-230547-5303/t.544/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_10086.root 2021-10-02 22:56:47 python_package_run(10569): command line exit code: 1 2021-10-02 22:56:47 python_package_run(10569): total runtime: 293.234 seconds task id 544 item 0e2e8207 failed: OUTPUT_MISS ('UL17_tttt_b4', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_10086.root', 'Events', 0, 94500) 0e2e8207 without result. resubmitting 0e2e8207 partly as 0e2e8207. 0 attempt(s) left. processing task id 541 item bfa76817 with 94500 events completed on d12chas580.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 141.0 MB, disk 330.0 MB, gpus: 0.0, runtime 295.695749 output: Using /cvmfs certificates. 2021-10-02 22:51:52 python_package_run(10426): setting up conda environment at /var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:51:52 python_package_run(10426): waiting for lock on directory /var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:51:52 python_package_run(10426): directory /var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:51:52 python_package_run(10426): activating environment 2021-10-02 22:51:52 python_package_run(10426): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_5228/worker-230547-5303/t.541/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_5228/worker-230547-5303/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_10091.root 2021-10-02 22:56:47 python_package_run(10426): command line exit code: 1 2021-10-02 22:56:47 python_package_run(10426): total runtime: 295.614 seconds task id 541 item bfa76817 failed: OUTPUT_MISS ('UL17_tttt_b4', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_10091.root', 'Events', 0, 94500) bfa76817 without result. resubmitting bfa76817 partly as bfa76817. 0 attempt(s) left. processing task id 550 item 46e81f1b with 113392 events completed on d12chas587.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.011, memory: 133.0 MB, disk 330.0 MB, gpus: 0.0, runtime 295.143612 output: Using /cvmfs certificates. 2021-10-02 22:51:56 python_package_run(32528): setting up conda environment at /var/condor/execute/dir_29058/worker-230547-29063/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:51:56 python_package_run(32528): waiting for lock on directory /var/condor/execute/dir_29058/worker-230547-29063/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:51:56 python_package_run(32528): directory /var/condor/execute/dir_29058/worker-230547-29063/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:51:56 python_package_run(32528): activating environment 2021-10-02 22:51:56 python_package_run(32528): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_29058/worker-230547-29063/t.550/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_29058/worker-230547-29063/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_29058/worker-230547-29063/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_29058/worker-230547-29063/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_29058/worker-230547-29063/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_29058/worker-230547-29063/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_29058/worker-230547-29063/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_29058/worker-230547-29063/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_29058/worker-230547-29063/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttlnuJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_8793.root 2021-10-02 22:56:51 python_package_run(32528): command line exit code: 1 2021-10-02 22:56:51 python_package_run(32528): total runtime: 295.014 seconds task id 550 item 46e81f1b failed: OUTPUT_MISS ('UL17_ttlnuJet_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_ttlnuJet_all22WCsStartPtCheckdim6TopMay20GST_run0/NAOD-00000_8793.root', 'Events', 113392, 226784) 46e81f1b without result. resubmitting 46e81f1b partly as 46e81f1b. 0 attempt(s) left. processing task id 536 item 3e4ad448 with 94500 events completed on qa-rtx6k-021.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.007, memory: 133.0 MB, disk 330.0 MB, gpus: 0.0, runtime 301.324497 output: Using /cvmfs certificates. 2021-10-02 22:51:51 python_package_run(161072): setting up conda environment at /var/condor/execute/dir_155375/worker-230547-155384/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:51:51 python_package_run(161072): waiting for lock on directory /var/condor/execute/dir_155375/worker-230547-155384/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 22:51:51 python_package_run(161072): directory /var/condor/execute/dir_155375/worker-230547-155384/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 22:51:51 python_package_run(161072): activating environment 2021-10-02 22:51:51 python_package_run(161072): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_155375/worker-230547-155384/t.536/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_155375/worker-230547-155384/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_155375/worker-230547-155384/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_155375/worker-230547-155384/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_155375/worker-230547-155384/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_155375/worker-230547-155384/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_155375/worker-230547-155384/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_155375/worker-230547-155384/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_155375/worker-230547-155384/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_10092.root 2021-10-02 22:56:52 python_package_run(161072): command line exit code: 1 2021-10-02 22:56:52 python_package_run(161072): total runtime: 301.222 seconds task id 536 item 3e4ad448 failed: OUTPUT_MISS ('UL17_tttt_b4', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch4/naodOnly_step/v1/nAOD_step_tttt_FourtopsMay3v1_run0/NAOD-00000_10092.root', 'Events', 0, 94500) 3e4ad448 without result. resubmitting 3e4ad448 partly as 3e4ad448. 0 attempt(s) left. processing task id 703 item a734f430 with 15000 events completed on d12chas547.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.152, memory: 480.0 MB, disk 340.0 MB, gpus: 0.0, runtime 169.984016 submitted processing task id 732 item f61c7ebf, with 116500 events processing task id 600 item 99de9cc3 with 83377 events completed on d12chas564.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.295, memory: 917.0 MB, disk 360.0 MB, gpus: 0.0, runtime 377.780711 processing task id 619 item cfec7462 with 83379 events completed on d12chas580.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.3, memory: 957.0 MB, disk 360.0 MB, gpus: 0.0, runtime 377.460444 submitted processing task id 733 item 07cbe790, with 123250 events processing task id 662 item a1b52282 with 79917 events completed on d12chas546.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.258, memory: 934.0 MB, disk 345.0 MB, gpus: 0.0, runtime 354.220741 submitted processing task id 734 item 63afbf13, with 123250 events processing task id 604 item 513cd0fb with 83657 events completed on d12chas545.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.289, memory: 903.0 MB, disk 360.0 MB, gpus: 0.0, runtime 382.062652 submitted processing task id 735 item 92205be0, with 114500 events processing task id 620 item 292d5022 with 83815 events completed on d12chas580.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.321, memory: 959.0 MB, disk 360.0 MB, gpus: 0.0, runtime 380.969151 submitted processing task id 736 item 943290a8, with 114500 events processing task id 627 item 71d1f727 with 83494 events completed on d12chas545.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.289, memory: 1914.0 MB, disk 360.0 MB, gpus: 0.0, runtime 378.200373 submitted processing task id 737 item 229fd5c3, with 118250 events processing task id 638 item e863d1d0 with 83657 events completed on d12chas564.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.316, memory: 952.0 MB, disk 360.0 MB, gpus: 0.0, runtime 377.793991 submitted processing task id 738 item a2edf762, with 118250 events processing task id 602 item cba5cfef with 83616 events completed on d12chas584.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.3, memory: 952.0 MB, disk 360.0 MB, gpus: 0.0, runtime 385.202038 submitted processing task id 739 item 1f0252fe, with 125000 events processing task id 623 item e98a8d57 with 83616 events completed on d12chas546.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.321, memory: 953.0 MB, disk 360.0 MB, gpus: 0.0, runtime 381.100011 processing task id 631 item fbbc3171 with 83835 events completed on d12chas546.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.3, memory: 1360.0 MB, disk 360.0 MB, gpus: 0.0, runtime 379.895074 submitted accumulation task id 740 item accum_5, with 10976843 events submitted processing task id 741 item 8f55b5f5, with 125000 events processing task id 626 item 11dc2e66 with 83499 events completed on d12chas545.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.3, memory: 952.0 MB, disk 360.0 MB, gpus: 0.0, runtime 381.582725 submitted processing task id 742 item 687ed7fb, with 116250 events submitted processing task id 743 item 82f8279f, with 116250 events processing task id 716 item e44ecb58 with 84000 events completed on d12chas547.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.3, memory: 1729.0 MB, disk 354.0 MB, gpus: 0.0, runtime 207.185552 submitted processing task id 744 item 9d9cc3b8, with 84334 events processing task id 643 item 66b5e67f with 83670 events completed on d12chas553.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.289, memory: 926.0 MB, disk 360.0 MB, gpus: 0.0, runtime 382.202262 processing task id 701 item ce1334d4 with 84000 events completed on d12chas546.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.311, memory: 963.0 MB, disk 354.0 MB, gpus: 0.0, runtime 246.540375 processing task id 621 item b860bd22 with 83614 events completed on d12chas580.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.305, memory: 931.0 MB, disk 360.0 MB, gpus: 0.0, runtime 386.537668 submitted processing task id 745 item d7a8bffd, with 84334 events submitted processing task id 746 item f8089763, with 84332 events submitted processing task id 747 item cafd272d, with 117750 events processing task id 630 item 49bf6c0d with 83833 events completed on d12chas546.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.3, memory: 955.0 MB, disk 360.0 MB, gpus: 0.0, runtime 385.521504 submitted processing task id 748 item cc00e90f, with 117750 events processing task id 636 item 847493d1 with 83776 events completed on d12chas564.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.316, memory: 931.0 MB, disk 360.0 MB, gpus: 0.0, runtime 385.380101 processing task id 665 item 215ae3cf with 84000 events completed on d12chas546.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.353, memory: 963.0 MB, disk 354.0 MB, gpus: 0.0, runtime 332.709636 processing task id 599 item c2543901 with 83776 events completed on d12chas564.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.321, memory: 955.0 MB, disk 360.0 MB, gpus: 0.0, runtime 393.011727 processing task id 667 item 20e357a8 with 84000 events completed on d12chas580.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.347, memory: 963.0 MB, disk 354.0 MB, gpus: 0.0, runtime 331.364763 processing task id 596 item b04b3218 with 83695 events completed on d12chas551.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.3, memory: 1844.0 MB, disk 360.0 MB, gpus: 0.0, runtime 393.896936 processing task id 698 item e09a20c8 with 84167 events completed on d12chas548.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.305, memory: 970.0 MB, disk 354.0 MB, gpus: 0.0, runtime 256.151021 processing task id 632 item d5beb3df with 83835 events completed on d12chas564.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.326, memory: 944.0 MB, disk 360.0 MB, gpus: 0.0, runtime 387.900646 processing task id 711 item 1fdd31e5 with 84334 events completed on d12chas564.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.316, memory: 1933.0 MB, disk 354.0 MB, gpus: 0.0, runtime 238.472696 processing task id 360 item d55cb577 with 119113 events completed on d12chas587.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.384, memory: 1228.0 MB, disk 353.0 MB, gpus: 0.0, runtime 305.559055 submitted processing task id 749 item 8851a4b7, with 119000 events submitted processing task id 750 item 8a70bd18, with 119000 events submitted processing task id 751 item 14d76627, with 121750 events submitted processing task id 752 item 4af59c14, with 121750 events submitted processing task id 753 item f098e7f0, with 86500 events submitted processing task id 754 item 60604d06, with 86500 events submitted processing task id 755 item 927b92f7, with 86500 events submitted processing task id 756 item 9f1fff0b, with 87500 events submitted processing task id 757 item d5b80e41, with 87500 events processing task id 589 item 9006fc01 with 112931 events completed on qa-rtx6k-021.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.284, memory: 2302.0 MB, disk 353.0 MB, gpus: 0.0, runtime 399.024112 submitted processing task id 758 item 6335eaf3, with 87500 events processing task id 684 item 27f14d17 with 84000 events completed on qa-rtx6k-021.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.242, memory: 964.0 MB, disk 354.0 MB, gpus: 0.0, runtime 271.232569 submitted processing task id 759 item fabde2cf, with 87834 events processing task id 671 item 92b263e7 with 114381 events completed on qa-rtx6k-021.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.284, memory: 1243.0 MB, disk 348.0 MB, gpus: 0.0, runtime 329.531452 processing task id 695 item 108cb9b2 with 112585 events completed on qa-rtx6k-021.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.279, memory: 1180.0 MB, disk 347.0 MB, gpus: 0.0, runtime 265.337762 submitted processing task id 760 item 7b4ae8c3, with 87834 events submitted processing task id 761 item 5234faa1, with 87832 events processing task id 563 item b404b6c2 with 94250 events completed on d12chas577.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.337, memory: 1708.0 MB, disk 355.0 MB, gpus: 0.0, runtime 415.223494 submitted processing task id 762 item d0884c49, with 87834 events processing task id 668 item f417645f with 84167 events completed on qa-rtx6k-021.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.237, memory: 974.0 MB, disk 354.0 MB, gpus: 0.0, runtime 330.547103 submitted processing task id 763 item 95ae1e7d, with 87834 events processing task id 591 item 2ed8cb5b with 112521 events completed on qa-rtx6k-021.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.3, memory: 1179.0 MB, disk 353.0 MB, gpus: 0.0, runtime 405.65816 processing task id 730 item 46e81f1b with 113392 events completed on d12chas587.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.555, memory: 2245.0 MB, disk 353.0 MB, gpus: 0.0, runtime 128.486258 submitted processing task id 764 item bca569ae, with 87832 events processing task id 652 item 62891c70 with 117734 events completed on d12chas551.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.389, memory: 1220.0 MB, disk 348.0 MB, gpus: 0.0, runtime 398.532532 processing task id 610 item 6430ab5b with 120609 events completed on d12chas580.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.374, memory: 1238.0 MB, disk 365.0 MB, gpus: 0.0, runtime 408.719063 submitted accumulation task id 765 item accum_6, with 2359882 events submitted processing task id 766 item 5e25dd81, with 87667 events processing task id 651 item b6d356f3 with 119758 events completed on d12chas564.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.368, memory: 2138.0 MB, disk 365.0 MB, gpus: 0.0, runtime 401.632084 processing task id 590 item 1369a390 with 112521 events completed on qa-rtx6k-021.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.289, memory: 1221.0 MB, disk 353.0 MB, gpus: 0.0, runtime 411.153034 processing task id 689 item 7b0b29c5 with 118118 events completed on qa-rtx6k-021.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.284, memory: 1264.0 MB, disk 349.0 MB, gpus: 0.0, runtime 295.705782 processing task id 702 item 533a94de with 84000 events completed on d12chas546.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.305, memory: 1934.0 MB, disk 354.0 MB, gpus: 0.0, runtime 267.195141 submitted processing task id 767 item 370e6287, with 87667 events submitted processing task id 768 item f3fe1ef1, with 87666 events processing task id 629 item cfe62b91 with 83499 events completed on d12chas545.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.279, memory: 944.0 MB, disk 360.0 MB, gpus: 0.0, runtime 405.324221 processing task id 622 item cf5b0133 with 120608 events completed on d12chas546.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.379, memory: 1535.0 MB, disk 365.0 MB, gpus: 0.0, runtime 407.869938 submitted processing task id 769 item 28e2b7b4, with 80000 events submitted processing task id 770 item b50d9fd9, with 124500 events submitted processing task id 771 item 06293753, with 124500 events submitted processing task id 772 item c7f4938c, with 87500 events submitted processing task id 773 item 531e1a5a, with 87500 events processing task id 677 item c36d2231 with 107092 events completed on d12chas547.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.342, memory: 2239.0 MB, disk 348.0 MB, gpus: 0.0, runtime 324.329201 submitted processing task id 774 item bf459734, with 87500 events processing task id 637 item b0a54562 with 117193 events completed on d12chas564.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.368, memory: 2264.0 MB, disk 364.0 MB, gpus: 0.0, runtime 406.170722 processing task id 679 item a5e80bc0 with 107091 events completed on d12chas553.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.363, memory: 1152.0 MB, disk 348.0 MB, gpus: 0.0, runtime 322.945678 processing task id 614 item ca160c19 with 116061 events completed on d12chas580.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.379, memory: 1328.0 MB, disk 364.0 MB, gpus: 0.0, runtime 412.888361 submitted processing task id 775 item 16bb2041, with 86167 events submitted processing task id 776 item a4f14797, with 86167 events submitted processing task id 777 item 6b1650d8, with 86166 events processing task id 641 item b57bb24a with 110778 events completed on d12chas551.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.368, memory: 1165.0 MB, disk 347.0 MB, gpus: 0.0, runtime 407.379393 processing task id 549 item ba70ab41 with 113080 events completed on d12chas547.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.368, memory: 1357.0 MB, disk 353.0 MB, gpus: 0.0, runtime 432.031353 processing task id 588 item c52427a6 with 119035 events completed on d12chas577.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.379, memory: 1277.0 MB, disk 353.0 MB, gpus: 0.0, runtime 416.918556 processing task id 681 item 422af9e3 with 84166 events completed on d12chas546.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.305, memory: 965.0 MB, disk 354.0 MB, gpus: 0.0, runtime 316.855329 processing task id 642 item 6a32324e with 119759 events completed on d12chas564.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.416, memory: 1226.0 MB, disk 365.0 MB, gpus: 0.0, runtime 409.607556 processing task id 566 item 14c0da6c with 108926 events completed on d12chas577.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.342, memory: 1216.0 MB, disk 352.0 MB, gpus: 0.0, runtime 430.834247 processing task id 567 item c9b9f8d0 with 119285 events completed on d12chas577.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.358, memory: 1240.0 MB, disk 353.0 MB, gpus: 0.0, runtime 431.157457 processing task id 572 item 983e90d7 with 108926 events completed on d12chas587.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.358, memory: 1344.0 MB, disk 352.0 MB, gpus: 0.0, runtime 431.211904 processing task id 573 item fd8b6091 with 112238 events completed on d12chas587.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.389, memory: 1174.0 MB, disk 353.0 MB, gpus: 0.0, runtime 429.629884 processing task id 607 item 2a311b79 with 83695 events completed on d12chas580.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.295, memory: 953.0 MB, disk 360.0 MB, gpus: 0.0, runtime 417.072394 processing task id 560 item e9636046 with 119248 events completed on d12chas587.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.405, memory: 1519.0 MB, disk 353.0 MB, gpus: 0.0, runtime 432.995381 processing task id 597 item e85b6676 with 119263 events completed on d12chas584.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.379, memory: 1246.0 MB, disk 354.0 MB, gpus: 0.0, runtime 419.313227 processing task id 608 item 311a4535 with 119263 events completed on d12chas584.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.405, memory: 1230.0 MB, disk 354.0 MB, gpus: 0.0, runtime 418.982123 processing task id 611 item 77e7bf20 with 119579 events completed on d12chas584.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.389, memory: 1463.0 MB, disk 353.0 MB, gpus: 0.0, runtime 416.648273 processing task id 624 item 4f82ab46 with 119308 events completed on d12chas546.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.405, memory: 1223.0 MB, disk 365.0 MB, gpus: 0.0, runtime 416.029261 submitted accumulation task id 778 item accum_7, with 2762490 events processing task id 633 item e335319a with 117734 events completed on d12chas546.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.379, memory: 1262.0 MB, disk 349.0 MB, gpus: 0.0, runtime 413.975062 processing task id 645 item ae0aef7c with 117193 events completed on d12chas551.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.395, memory: 1219.0 MB, disk 364.0 MB, gpus: 0.0, runtime 409.828671 processing task id 609 item ec90fb76 with 83695 events completed on d12chas564.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.311, memory: 953.0 MB, disk 360.0 MB, gpus: 0.0, runtime 421.390451 processing task id 579 item c712bfc5 with 119522 events completed on d12chas589.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.358, memory: 1232.0 MB, disk 353.0 MB, gpus: 0.0, runtime 425.985171 processing task id 649 item 646abe8b with 83499 events completed on d12chas564.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.311, memory: 1090.0 MB, disk 360.0 MB, gpus: 0.0, runtime 411.383513 processing task id 655 item 46292915 with 118056 events completed on d12chas564.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.374, memory: 1264.0 MB, disk 348.0 MB, gpus: 0.0, runtime 402.752317 processing task id 553 item 044f2f1f with 119113 events completed on d12chas587.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.389, memory: 1228.0 MB, disk 353.0 MB, gpus: 0.0, runtime 438.297688 processing task id 580 item d84b939c with 119232 events completed on d12chas589.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.374, memory: 1239.0 MB, disk 353.0 MB, gpus: 0.0, runtime 427.222553 processing task id 673 item dbd84d3a with 117841 events completed on d12chas582.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.342, memory: 1262.0 MB, disk 349.0 MB, gpus: 0.0, runtime 345.783284 processing task id 557 item 5bee00c9 with 118381 events completed on d12chas552.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.374, memory: 1262.0 MB, disk 353.0 MB, gpus: 0.0, runtime 438.13915 processing task id 571 item e2a5c979 with 119598 events completed on d12chas548.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.384, memory: 1234.0 MB, disk 353.0 MB, gpus: 0.0, runtime 437.544972 processing task id 581 item 5b67d2af with 116056 events completed on d12chas584.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.416, memory: 2416.0 MB, disk 353.0 MB, gpus: 0.0, runtime 428.172773 processing task id 570 item 57830eba with 119218 events completed on d12chas587.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.405, memory: 1230.0 MB, disk 353.0 MB, gpus: 0.0, runtime 438.669287 processing task id 568 item fa3eb9f8 with 119157 events completed on d12chas577.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.358, memory: 2287.0 MB, disk 353.0 MB, gpus: 0.0, runtime 439.244659 processing task id 594 item 6d20edce with 124612 events completed on d12chas564.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.379, memory: 2407.0 MB, disk 366.0 MB, gpus: 0.0, runtime 425.045742 processing task id 683 item 0ba00551 with 107703 events completed on d12chas584.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.353, memory: 2271.0 MB, disk 347.0 MB, gpus: 0.0, runtime 313.803875 processing task id 551 item 2da82cb2 with 113079 events completed on d12chas587.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.395, memory: 1191.0 MB, disk 353.0 MB, gpus: 0.0, runtime 442.297768 processing task id 615 item 22d5ed52 with 116061 events completed on d12chas546.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.389, memory: 2463.0 MB, disk 364.0 MB, gpus: 0.0, runtime 425.685067 processing task id 659 item a718850f with 115140 events completed on d12chas548.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.384, memory: 1223.0 MB, disk 349.0 MB, gpus: 0.0, runtime 404.021421 processing task id 612 item e4a9c968 with 109272 events completed on d12chas546.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.4, memory: 2171.0 MB, disk 352.0 MB, gpus: 0.0, runtime 426.545023 processing task id 693 item c6ad0ecc with 111303 events completed on d12chas551.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.395, memory: 1170.0 MB, disk 348.0 MB, gpus: 0.0, runtime 292.038181 processing task id 694 item 84c63233 with 84000 events completed on d12chas580.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.3, memory: 976.0 MB, disk 354.0 MB, gpus: 0.0, runtime 285.552059 processing task id 705 item ed0f0f1d with 84000 events completed on d12chas589.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.3, memory: 974.0 MB, disk 354.0 MB, gpus: 0.0, runtime 271.373499 processing task id 714 item 3c0912a7 with 117250 events completed on d12chas551.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.374, memory: 1215.0 MB, disk 357.0 MB, gpus: 0.0, runtime 246.294394 processing task id 715 item 9b6c849e with 117250 events completed on d12chas580.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.426, memory: 2481.0 MB, disk 357.0 MB, gpus: 0.0, runtime 255.512058 submitted accumulation task id 779 item accum_8, with 2787965 events processing task id 720 item e5241d59 with 94500 events completed on d12chas580.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.501, memory: 2045.0 MB, disk 354.0 MB, gpus: 0.0, runtime 148.552506 processing task id 723 item d4f5ccfc with 94500 events completed on d12chas580.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.517, memory: 1376.0 MB, disk 354.0 MB, gpus: 0.0, runtime 150.058237 processing task id 728 item 0e2e8207 with 94500 events completed on d12chas580.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.501, memory: 1110.0 MB, disk 354.0 MB, gpus: 0.0, runtime 152.190047 processing task id 576 item 6cf8a98a with 94500 events completed on d12chas552.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.395, memory: 2176.0 MB, disk 354.0 MB, gpus: 0.0, runtime 435.877827 processing task id 731 item 3e4ad448 with 94500 events completed on qa-rtx6k-021.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.395, memory: 1965.0 MB, disk 354.0 MB, gpus: 0.0, runtime 142.574975 accumulating task id 740 item accum_5 with 10976843 events completed on qa-rtx6k-021.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.996, memory: 1890.0 MB, disk 1282.0 MB, gpus: 0.0, runtime 20.665951 processing task id 562 item fae745f2 with 119217 events completed on d12chas548.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.395, memory: 1254.0 MB, disk 353.0 MB, gpus: 0.0, runtime 446.552948 processing task id 575 item 97ee91e3 with 117530 events completed on d12chas548.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.416, memory: 2495.0 MB, disk 353.0 MB, gpus: 0.0, runtime 437.853528 processing task id 663 item 45204ecc with 94500 events completed on d12chas547.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.358, memory: 1106.0 MB, disk 355.0 MB, gpus: 0.0, runtime 392.786542 processing task id 710 item 5e7035ac with 123250 events completed on d12chas553.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.416, memory: 1262.0 MB, disk 358.0 MB, gpus: 0.0, runtime 283.771932 processing task id 721 item 01a58bfd with 94500 events completed on d12chas580.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.515, memory: 2186.0 MB, disk 355.0 MB, gpus: 0.0, runtime 158.804291 processing task id 725 item 27aaaee0 with 94500 events completed on d12chas580.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.466, memory: 1934.0 MB, disk 355.0 MB, gpus: 0.0, runtime 158.958952 submitted processing task id 780 item d1897c32, with 87500 events submitted processing task id 781 item a0e334ff, with 87500 events submitted processing task id 782 item d4b11dea, with 87500 events submitted processing task id 783 item 405409f6, with 87500 events submitted processing task id 784 item e1a7b8b6, with 87500 events submitted processing task id 785 item 0c7f8184, with 87500 events submitted processing task id 786 item c54dc74b, with 120000 events submitted processing task id 787 item 9289b12b, with 120000 events submitted processing task id 788 item 2d4c025e, with 84167 events submitted processing task id 789 item 7d55cb30, with 84167 events processing task id 653 item 7a97b475 with 83669 events completed on d12chas553.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.316, memory: 953.0 MB, disk 360.0 MB, gpus: 0.0, runtime 432.678915 submitted processing task id 790 item 52d04ef0, with 84166 events submitted processing task id 791 item 8a4d5aa8, with 119250 events submitted processing task id 792 item 11afe6d4, with 119250 events submitted processing task id 793 item ed20163e, with 122500 events submitted processing task id 794 item 554db316, with 122500 events submitted processing task id 795 item e48c8b6d, with 85667 events submitted processing task id 796 item 859a7cc9, with 85667 events submitted processing task id 797 item 1d28493d, with 85666 events submitted processing task id 798 item d8cf3acf, with 83834 events submitted processing task id 799 item 6b40cd9f, with 83834 events processing task id 712 item 5ea15449 with 116500 events completed on d12chas547.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.582, memory: 1484.0 MB, disk 357.0 MB, gpus: 0.0, runtime 130.017548 submitted processing task id 800 item 11a1f651, with 83832 events submitted processing task id 801 item 16617b8b, with 118750 events submitted processing task id 802 item 03b3de27, with 118750 events processing task id 696 item 61d7ae8d with 107507 events completed on d12chas547.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.353, memory: 1140.0 MB, disk 348.0 MB, gpus: 0.0, runtime 310.693607 processing task id 686 item a4d1e7a1 with 113673 events completed on d12chas551.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.353, memory: 2387.0 MB, disk 348.0 MB, gpus: 0.0, runtime 333.638384 processing task id 697 item c596506b with 107506 events completed on d12chas548.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.363, memory: 1143.0 MB, disk 348.0 MB, gpus: 0.0, runtime 310.941063 processing task id 593 item 5e3d8b77 with 123773 events completed on d12chas545.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.379, memory: 1255.0 MB, disk 365.0 MB, gpus: 0.0, runtime 449.749541 processing task id 595 item 4dbb2b4e with 123774 events completed on d12chas564.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.395, memory: 2358.0 MB, disk 365.0 MB, gpus: 0.0, runtime 452.552614 processing task id 554 item 397c23ee with 112549 events completed on d12chas552.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.405, memory: 1330.0 MB, disk 352.0 MB, gpus: 0.0, runtime 468.174235 processing task id 657 item 2c090759 with 106862 events completed on d12chas548.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.347, memory: 1135.0 MB, disk 347.0 MB, gpus: 0.0, runtime 432.05158 processing task id 656 item b6954b26 with 106863 events completed on d12chas564.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.358, memory: 1148.0 MB, disk 348.0 MB, gpus: 0.0, runtime 433.118205 processing task id 674 item 66758e11 with 113672 events completed on d12chas545.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.416, memory: 2436.0 MB, disk 348.0 MB, gpus: 0.0, runtime 376.393661 processing task id 682 item 5085a027 with 111956 events completed on qa-rtx6k-021.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.284, memory: 2379.0 MB, disk 348.0 MB, gpus: 0.0, runtime 358.926267 processing task id 603 item f8fdfac1 with 117814 events completed on d12chas548.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.384, memory: 1252.0 MB, disk 364.0 MB, gpus: 0.0, runtime 459.711493 submitted accumulation task id 803 item accum_9, with 13538958 events accumulating task id 765 item accum_6 with 2359882 events completed on qa-rtx6k-021.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.996, memory: 1978.0 MB, disk 1166.0 MB, gpus: 0.0, runtime 22.317297 accumulating task id 779 item accum_8 with 2787965 events completed on d12chas584.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.998, memory: 1646.0 MB, disk 1119.0 MB, gpus: 0.0, runtime 18.35255 accumulating task id 778 item accum_7 with 2762490 events completed on d12chas552.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.998, memory: 1598.0 MB, disk 1147.0 MB, gpus: 0.0, runtime 19.156977 processing task id 756 item 9f1fff0b with 87500 events completed on qa-rtx6k-021.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.551, memory: 958.0 MB, disk 334.0 MB, gpus: 0.0, runtime 76.699844 processing task id 763 item 95ae1e7d with 87834 events completed on qa-rtx6k-021.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.64, memory: 957.0 MB, disk 334.0 MB, gpus: 0.0, runtime 72.275317 processing task id 759 item fabde2cf with 87834 events completed on qa-rtx6k-021.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.585, memory: 956.0 MB, disk 334.0 MB, gpus: 0.0, runtime 77.270379 processing task id 761 item 5234faa1 with 87832 events completed on qa-rtx6k-021.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.547, memory: 957.0 MB, disk 334.0 MB, gpus: 0.0, runtime 80.799976 processing task id 766 item 5e25dd81 with 87667 events completed on qa-rtx6k-021.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.596, memory: 955.0 MB, disk 334.0 MB, gpus: 0.0, runtime 73.131769 processing task id 762 item d0884c49 with 87834 events completed on d12chas577.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.645, memory: 955.0 MB, disk 335.0 MB, gpus: 0.0, runtime 82.663112 processing task id 738 item a2edf762 with 118250 events completed on d12chas564.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.657, memory: 1263.0 MB, disk 358.0 MB, gpus: 0.0, runtime 109.873112 processing task id 732 item f61c7ebf with 116500 events completed on d12chas564.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.632, memory: 2274.0 MB, disk 357.0 MB, gpus: 0.0, runtime 117.489541 processing task id 515 item 6d077bb2 with 84167 events completed on d12chas553.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.622, memory: 967.0 MB, disk 354.0 MB, gpus: 0.0, runtime 105.807018 processing task id 772 item c7f4938c with 87500 events completed on d12chas547.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.661, memory: 1966.0 MB, disk 334.0 MB, gpus: 0.0, runtime 84.284849 processing task id 735 item 92205be0 with 114500 events completed on d12chas545.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.639, memory: 1196.0 MB, disk 357.0 MB, gpus: 0.0, runtime 116.251891 processing task id 737 item 229fd5c3 with 118250 events completed on d12chas545.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.62, memory: 1261.0 MB, disk 357.0 MB, gpus: 0.0, runtime 115.098098 processing task id 764 item bca569ae with 87832 events completed on d12chas587.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.666, memory: 1763.0 MB, disk 334.0 MB, gpus: 0.0, runtime 91.58419 accumulating task id 803 item accum_9 with 13538958 events completed on d12chas552.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.998, memory: 1982.0 MB, disk 1432.0 MB, gpus: 0.0, runtime 24.57951 processing task id 743 item 82f8279f with 116250 events completed on d12chas545.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.644, memory: 1207.0 MB, disk 357.0 MB, gpus: 0.0, runtime 115.758912 processing task id 773 item 531e1a5a with 87500 events completed on d12chas546.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.685, memory: 960.0 MB, disk 334.0 MB, gpus: 0.0, runtime 92.639441 processing task id 774 item bf459734 with 87500 events completed on d12chas553.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.634, memory: 1730.0 MB, disk 334.0 MB, gpus: 0.0, runtime 90.675369 processing task id 748 item cc00e90f with 117750 events completed on d12chas548.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.629, memory: 1525.0 MB, disk 357.0 MB, gpus: 0.0, runtime 113.663756 processing task id 742 item 687ed7fb with 116250 events completed on d12chas546.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.634, memory: 1533.0 MB, disk 358.0 MB, gpus: 0.0, runtime 120.837157 processing task id 733 item 07cbe790 with 123250 events completed on d12chas580.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.6, memory: 1262.0 MB, disk 358.0 MB, gpus: 0.0, runtime 133.006942 processing task id 768 item f3fe1ef1 with 87666 events completed on d12chas545.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.509, memory: 958.0 MB, disk 334.0 MB, gpus: 0.0, runtime 106.951438 processing task id 739 item 1f0252fe with 125000 events completed on d12chas546.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.648, memory: 1283.0 MB, disk 358.0 MB, gpus: 0.0, runtime 134.792836 submitted accumulation task id 804 item accum_10, with 23563961 events processing task id 518 item cea5f55f with 122000 events completed on d12chas584.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.578, memory: 1272.0 MB, disk 357.0 MB, gpus: 0.0, runtime 139.070402 processing task id 767 item 370e6287 with 87667 events completed on d12chas564.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.528, memory: 957.0 MB, disk 334.0 MB, gpus: 0.0, runtime 113.808978 processing task id 758 item 6335eaf3 with 87500 events completed on qa-rtx6k-021.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.296, memory: 1696.0 MB, disk 334.0 MB, gpus: 0.0, runtime 144.284603 processing task id 465 item 8829337a with 83776 events completed on d12chas546.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.401, memory: 942.0 MB, disk 360.0 MB, gpus: 0.0, runtime 149.574297 processing task id 746 item f8089763 with 84332 events completed on d12chas551.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.373, memory: 967.0 MB, disk 354.0 MB, gpus: 0.0, runtime 155.265768 accumulating task id 804 item accum_10 with 23563961 events completed on d12chas546.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.998, memory: 3421.0 MB, disk 1839.0 MB, gpus: 0.0, runtime 34.326231 processing task id 585 item 76a6d86f with 117615 events completed on d12chas584.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.284, memory: 1218.0 MB, disk 353.0 MB, gpus: 0.0, runtime 608.944554 processing task id 771 item 06293753 with 124500 events completed on d12chas545.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.711, memory: 1264.0 MB, disk 335.0 MB, gpus: 0.0, runtime 98.661528 processing task id 769 item 28e2b7b4 with 80000 events completed on d12chas589.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2750.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.263, memory: 880.0 MB, disk 334.0 MB, gpus: 0.0, runtime 249.155577 processing task id 783 item 405409f6 with 87500 events completed on d12chas552.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2750.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.279, memory: 956.0 MB, disk 334.0 MB, gpus: 0.0, runtime 248.393503 processing task id 796 item 859a7cc9 with 85667 events completed on d12chas587.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2750.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.268, memory: 1498.0 MB, disk 334.0 MB, gpus: 0.0, runtime 246.927676 processing task id 789 item 7d55cb30 with 84167 events completed on d12chas548.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2750.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.284, memory: 932.0 MB, disk 334.0 MB, gpus: 0.0, runtime 249.019161 processing task id 777 item 6b1650d8 with 86166 events completed on d12chas548.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2750.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.279, memory: 945.0 MB, disk 334.0 MB, gpus: 0.0, runtime 254.027561 processing task id 782 item d4b11dea with 87500 events completed on d12chas552.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2750.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.284, memory: 1858.0 MB, disk 334.0 MB, gpus: 0.0, runtime 250.988655 processing task id 781 item a0e334ff with 87500 events completed on d12chas546.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2750.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.284, memory: 1299.0 MB, disk 334.0 MB, gpus: 0.0, runtime 252.029387 processing task id 625 item 6e4f5411 with 119307 events completed on d12chas547.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.347, memory: 1223.0 MB, disk 364.0 MB, gpus: 0.0, runtime 256.858916 processing task id 795 item e48c8b6d with 85667 events completed on d12chas587.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2750.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.3, memory: 939.0 MB, disk 334.0 MB, gpus: 0.0, runtime 250.773405 processing task id 780 item d1897c32 with 87500 events completed on d12chas546.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2750.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.295, memory: 1930.0 MB, disk 334.0 MB, gpus: 0.0, runtime 252.89558 processing task id 797 item 1d28493d with 85666 events completed on d12chas587.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2750.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.289, memory: 1781.0 MB, disk 334.0 MB, gpus: 0.0, runtime 257.394942 processing task id 445 item e6f01087 with 103084 events completed on d12chas580.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2750.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.342, memory: 2101.0 MB, disk 362.0 MB, gpus: 0.0, runtime 265.139221 processing task id 788 item 2d4c025e with 84167 events completed on d12chas577.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2750.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.263, memory: 1234.0 MB, disk 334.0 MB, gpus: 0.0, runtime 266.355873 processing task id 574 item d8131585 with 117530 events completed on d12chas548.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.368, memory: 2174.0 MB, disk 353.0 MB, gpus: 0.0, runtime 274.345332 processing task id 586 item 566bf5de with 112239 events completed on d12chas580.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.405, memory: 2476.0 MB, disk 352.0 MB, gpus: 0.0, runtime 274.074085 processing task id 577 item fdbf65b4 with 116057 events completed on d12chas548.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.384, memory: 1205.0 MB, disk 353.0 MB, gpus: 0.0, runtime 277.11997 processing task id 790 item 52d04ef0 with 84166 events completed on d12chas584.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2750.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.289, memory: 932.0 MB, disk 335.0 MB, gpus: 0.0, runtime 270.10618 submitted accumulation task id 805 item accum_11, with 25865234 events processing task id 787 item 9289b12b with 120000 events completed on d12chas577.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2750.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.337, memory: 2512.0 MB, disk 335.0 MB, gpus: 0.0, runtime 272.226637 processing task id 592 item 8b2a2c8c with 119578 events completed on d12chas548.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.411, memory: 1231.0 MB, disk 353.0 MB, gpus: 0.0, runtime 278.494078 processing task id 770 item b50d9fd9 with 124500 events completed on d12chas589.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2750.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.358, memory: 1635.0 MB, disk 336.0 MB, gpus: 0.0, runtime 279.067319 processing task id 802 item 03b3de27 with 118750 events completed on d12chas547.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2750.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.337, memory: 2233.0 MB, disk 336.0 MB, gpus: 0.0, runtime 272.626114 processing task id 792 item 11afe6d4 with 119250 events completed on d12chas587.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2750.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.358, memory: 1891.0 MB, disk 335.0 MB, gpus: 0.0, runtime 274.043732 processing task id 727 item 43010f60 with 94500 events completed on d12chas547.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.268, memory: 1126.0 MB, disk 354.0 MB, gpus: 0.0, runtime 448.355924 processing task id 561 item 61a912bc with 138451 events completed on d12chas577.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.4, memory: 2968.0 MB, disk 355.0 MB, gpus: 0.0, runtime 283.695427 processing task id 749 item 8851a4b7 with 119000 events completed on d12chas580.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2750.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.374, memory: 1205.0 MB, disk 335.0 MB, gpus: 0.0, runtime 286.898119 processing task id 751 item 14d76627 with 121750 events completed on d12chas580.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2750.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.363, memory: 2492.0 MB, disk 335.0 MB, gpus: 0.0, runtime 287.816689 processing task id 752 item 4af59c14 with 121750 events completed on d12chas580.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2750.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.363, memory: 1240.0 MB, disk 335.0 MB, gpus: 0.0, runtime 289.2003 processing task id 793 item ed20163e with 122500 events completed on d12chas587.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2750.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.342, memory: 2514.0 MB, disk 335.0 MB, gpus: 0.0, runtime 303.114174 processing task id 647 item eed4e174 with 118914 events completed on qa-rtx6k-021.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.274, memory: 2306.0 MB, disk 365.0 MB, gpus: 0.0, runtime 278.316822 processing task id 794 item 554db316 with 122500 events completed on d12chas587.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2750.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.347, memory: 1211.0 MB, disk 335.0 MB, gpus: 0.0, runtime 305.924763 accumulating task id 805 item accum_11 with 25865234 events completed on d12chas547.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.998, memory: 3050.0 MB, disk 1357.0 MB, gpus: 0.0, runtime 29.98244 processing task id 734 item 63afbf13 with 123250 events completed on qa-rtx6k-021.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.007, memory: 133.0 MB, disk 330.0 MB, gpus: 0.0, runtime 301.297762 output: Using /cvmfs certificates. 2021-10-02 23:00:30 python_package_run(164232): setting up conda environment at /var/condor/execute/dir_155377/worker-230547-155406/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 23:00:30 python_package_run(164232): waiting for lock on directory /var/condor/execute/dir_155377/worker-230547-155406/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 23:00:30 python_package_run(164232): directory /var/condor/execute/dir_155377/worker-230547-155406/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 23:00:30 python_package_run(164232): activating environment 2021-10-02 23:00:30 python_package_run(164232): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_155377/worker-230547-155406/t.734/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_155377/worker-230547-155406/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_155377/worker-230547-155406/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_155377/worker-230547-155406/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_155377/worker-230547-155406/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_155377/worker-230547-155406/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_155377/worker-230547-155406/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_155377/worker-230547-155406/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_155377/worker-230547-155406/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_tllq4fNoSchanWNoHiggs0p_all22WCsStartPtCheckV2dim6TopMay20GST_run0/NAOD-00000_15898.root 2021-10-02 23:05:31 python_package_run(164232): command line exit code: 1 2021-10-02 23:05:31 python_package_run(164232): total runtime: 301.199 seconds task id 734 item 63afbf13 failed: OUTPUT_MISS ('UL17_tllq_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_tllq4fNoSchanWNoHiggs0p_all22WCsStartPtCheckV2dim6TopMay20GST_run0/NAOD-00000_15898.root', 'Events', 123250, 246500) 63afbf13 without result. resubmitting 63afbf13 partly as 63afbf13. 1 attempt(s) left. processing task id 736 item 943290a8 with 114500 events completed on d12chas553.crc.nd.edu. return code 1 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.011, memory: 132.0 MB, disk 330.0 MB, gpus: 0.0, runtime 303.139469 output: Using /cvmfs certificates. 2021-10-02 23:00:39 python_package_run(19085): setting up conda environment at /var/condor/execute/dir_12055/worker-230547-12064/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 23:00:39 python_package_run(19085): waiting for lock on directory /var/condor/execute/dir_12055/worker-230547-12064/full_env_d0db474c_HEAD.tar.gz-env 2021-10-02 23:00:39 python_package_run(19085): directory /var/condor/execute/dir_12055/worker-230547-12064/full_env_d0db474c_HEAD.tar.gz-env is not empty. Not expanding environment file again 2021-10-02 23:00:39 python_package_run(19085): activating environment 2021-10-02 23:00:39 python_package_run(19085): executing command line: python fn_wrapper function.p args.p output.p Traceback (most recent call last): File "/var/condor/execute/dir_12055/worker-230547-12064/t.736/fn_wrapper", line 17, in pickle_out = exec_function(*exec_args) File "/var/condor/execute/dir_12055/worker-230547-12064/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/coffea/processor/executor.py", line 184, in __call__ out = self.function(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 941, in automatic_retries raise e File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 929, in automatic_retries return func(*args, **kwargs) File "/afs/crc.nd.edu/user/k/kmohrman/miniconda3/envs/coffea-env3/lib/python3.9/site-packages/coffea/processor/executor.py", line 1101, in _work_function filecontext = uproot.open( File "/var/condor/execute/dir_12055/worker-230547-12064/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 149, in open file = ReadOnlyFile( File "/var/condor/execute/dir_12055/worker-230547-12064/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/reading.py", line 589, in __init__ self._source = Source( File "/var/condor/execute/dir_12055/worker-230547-12064/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 266, in __init__ self._open() File "/var/condor/execute/dir_12055/worker-230547-12064/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 269, in _open self._resource = XRootDResource(self._file_path, self._timeout) File "/var/condor/execute/dir_12055/worker-230547-12064/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 82, in __init__ self._open() File "/var/condor/execute/dir_12055/worker-230547-12064/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 91, in _open self._xrd_error(status) File "/var/condor/execute/dir_12055/worker-230547-12064/full_env_d0db474c_HEAD.tar.gz-env/lib/python3.9/site-packages/uproot/source/xrootd.py", line 117, in _xrd_error raise OSError( OSError: XRootD error: [ERROR] Operation expired in file root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_tllq4fNoSchanWNoHiggs0p_all22WCsStartPtCheckV2dim6TopMay20GST_run0/NAOD-00000_14736.root 2021-10-02 23:05:42 python_package_run(19085): command line exit code: 1 2021-10-02 23:05:42 python_package_run(19085): total runtime: 302.998 seconds task id 736 item 943290a8 failed: OUTPUT_MISS ('UL17_tllq_b1', 'root://deepthought.crc.nd.edu//store/user/kmohrman/FullProduction/FullR2/UL17/Round1/Batch1/naodOnly_step/v2/nAOD_step_tllq4fNoSchanWNoHiggs0p_all22WCsStartPtCheckV2dim6TopMay20GST_run0/NAOD-00000_14736.root', 'Events', 114500, 229000) 943290a8 without result. resubmitting 943290a8 partly as 943290a8. 1 attempt(s) left. processing task id 741 item 8f55b5f5 with 125000 events completed on d12chas548.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.384, memory: 1277.0 MB, disk 358.0 MB, gpus: 0.0, runtime 348.415891 processing task id 747 item cafd272d with 117750 events completed on d12chas580.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.411, memory: 1251.0 MB, disk 357.0 MB, gpus: 0.0, runtime 360.667291 processing task id 664 item be614941 with 84334 events completed on d12chas577.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.211, memory: 1795.0 MB, disk 354.0 MB, gpus: 0.0, runtime 833.195327 processing task id 617 item d11433bf with 83379 events completed on d12chas580.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.195, memory: 1907.0 MB, disk 360.0 MB, gpus: 0.0, runtime 877.551536 processing task id 644 item d9b24637 with 83670 events completed on d12chas553.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.195, memory: 1604.0 MB, disk 360.0 MB, gpus: 0.0, runtime 883.910293 processing task id 676 item 7f44a48f with 111633 events completed on d12chas548.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.221, memory: 2461.0 MB, disk 348.0 MB, gpus: 0.0, runtime 823.104894 processing task id 639 item f38ff8ed with 109130 events completed on d12chas551.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.216, memory: 1154.0 MB, disk 348.0 MB, gpus: 0.0, runtime 935.189801 processing task id 559 item 8e4ec4f4 with 94500 events completed on d12chas552.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.216, memory: 1164.0 MB, disk 354.0 MB, gpus: 0.0, runtime 961.871436 processing task id 691 item e15053f0 with 113889 events completed on qa-rtx6k-021.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.142, memory: 1193.0 MB, disk 348.0 MB, gpus: 0.0, runtime 829.127202 processing task id 618 item f2c2b43a with 113449 events completed on d12chas580.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.216, memory: 1192.0 MB, disk 364.0 MB, gpus: 0.0, runtime 947.910209 processing task id 598 item 81e201b9 with 113449 events completed on d12chas580.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.211, memory: 2311.0 MB, disk 364.0 MB, gpus: 0.0, runtime 950.799117 submitted accumulation task id 808 item accum_12, with 28576860 events processing task id 634 item 2d531ad7 with 114922 events completed on d12chas545.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.211, memory: 1203.0 MB, disk 364.0 MB, gpus: 0.0, runtime 945.3848 processing task id 552 item 0b1bcd41 with 94250 events completed on d12chas577.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.189, memory: 1601.0 MB, disk 354.0 MB, gpus: 0.0, runtime 974.50489 processing task id 605 item 34c2dd95 with 103085 events completed on d12chas546.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.184, memory: 1490.0 MB, disk 363.0 MB, gpus: 0.0, runtime 961.401042 processing task id 628 item 082610bb with 83494 events completed on d12chas545.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.158, memory: 1710.0 MB, disk 360.0 MB, gpus: 0.0, runtime 959.036565 processing task id 722 item 7d52b268 with 94500 events completed on d12chas580.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.205, memory: 1170.0 MB, disk 355.0 MB, gpus: 0.0, runtime 691.369225 processing task id 719 item 056212f9 with 94500 events completed on d12chas564.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.195, memory: 1858.0 MB, disk 355.0 MB, gpus: 0.0, runtime 692.040776 processing task id 606 item 8e8179e8 with 124612 events completed on d12chas564.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.221, memory: 1263.0 MB, disk 365.0 MB, gpus: 0.0, runtime 969.489649 processing task id 718 item 34a284aa with 94500 events completed on d12chas547.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.205, memory: 1166.0 MB, disk 355.0 MB, gpus: 0.0, runtime 696.431578 processing task id 726 item 3e008e90 with 94250 events completed on d12chas580.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.232, memory: 1168.0 MB, disk 355.0 MB, gpus: 0.0, runtime 696.369614 processing task id 724 item 5fcba9fa with 94250 events completed on d12chas580.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.221, memory: 1167.0 MB, disk 354.0 MB, gpus: 0.0, runtime 711.888186 processing task id 535 item 8e6250aa with 84000 events completed on d12chas587.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.189, memory: 1932.0 MB, disk 354.0 MB, gpus: 0.0, runtime 932.524519 accumulating task id 808 item accum_12 with 28576860 events completed on qa-rtx6k-021.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.996, memory: 3014.0 MB, disk 1513.0 MB, gpus: 0.0, runtime 25.269047 processing task id 713 item eed89c36 with 84332 events completed on d12chas553.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.189, memory: 971.0 MB, disk 354.0 MB, gpus: 0.0, runtime 858.404712 processing task id 680 item 8f01a6db with 111956 events completed on d12chas547.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.205, memory: 1187.0 MB, disk 348.0 MB, gpus: 0.0, runtime 929.975011 processing task id 672 item c18ecd8e with 114381 events completed on d12chas548.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.232, memory: 2380.0 MB, disk 348.0 MB, gpus: 0.0, runtime 951.86452 processing task id 654 item 9abfe290 with 118018 events completed on d12chas546.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.211, memory: 1444.0 MB, disk 348.0 MB, gpus: 0.0, runtime 1012.699498 processing task id 650 item 52f122d1 with 110778 events completed on d12chas551.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.221, memory: 2443.0 MB, disk 348.0 MB, gpus: 0.0, runtime 1023.054923 processing task id 687 item 6894b4b3 with 107702 events completed on d12chas553.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.232, memory: 2352.0 MB, disk 347.0 MB, gpus: 0.0, runtime 919.603013 processing task id 669 item 4f4fcba7 with 117840 events completed on d12chas548.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.232, memory: 1628.0 MB, disk 349.0 MB, gpus: 0.0, runtime 967.431154 processing task id 455 item 3877ac8c with 83657 events completed on d12chas546.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.205, memory: 952.0 MB, disk 360.0 MB, gpus: 0.0, runtime 642.829803 processing task id 646 item 3809c2f8 with 118017 events completed on d12chas564.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.242, memory: 1535.0 MB, disk 348.0 MB, gpus: 0.0, runtime 1034.706169 processing task id 666 item 886e5fe0 with 84000 events completed on d12chas580.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.168, memory: 971.0 MB, disk 354.0 MB, gpus: 0.0, runtime 982.431743 processing task id 658 item f4920729 with 109131 events completed on d12chas546.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.216, memory: 1119.0 MB, disk 348.0 MB, gpus: 0.0, runtime 1023.376026 processing task id 700 item e742823e with 84000 events completed on d12chas589.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.174, memory: 1797.0 MB, disk 354.0 MB, gpus: 0.0, runtime 905.022336 processing task id 692 item 28cc63de with 113888 events completed on qa-rtx6k-021.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.195, memory: 2243.0 MB, disk 348.0 MB, gpus: 0.0, runtime 916.723579 submitted accumulation task id 809 item accum_13, with 31010923 events processing task id 690 item 47c87f57 with 117969 events completed on qa-rtx6k-021.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.189, memory: 1222.0 MB, disk 348.0 MB, gpus: 0.0, runtime 935.574556 processing task id 661 item 7c4db65b with 117961 events completed on d12chas546.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.232, memory: 2352.0 MB, disk 348.0 MB, gpus: 0.0, runtime 1023.973315 processing task id 556 item 7434bd8e with 115079 events completed on d12chas552.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.221, memory: 1197.0 MB, disk 353.0 MB, gpus: 0.0, runtime 1071.506346 processing task id 678 item 3cd4a28f with 84167 events completed on d12chas545.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.158, memory: 967.0 MB, disk 354.0 MB, gpus: 0.0, runtime 981.249467 processing task id 587 item 96d6a8ff with 119035 events completed on d12chas589.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.226, memory: 1228.0 MB, disk 354.0 MB, gpus: 0.0, runtime 1073.729782 processing task id 601 item f6cc5187 with 109272 events completed on d12chas580.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.2, memory: 1159.0 MB, disk 352.0 MB, gpus: 0.0, runtime 1071.545864 processing task id 578 item b471bce2 with 112549 events completed on d12chas589.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.216, memory: 1183.0 MB, disk 352.0 MB, gpus: 0.0, runtime 1076.723289 processing task id 569 item 8d469b3d with 119285 events completed on d12chas577.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.237, memory: 1247.0 MB, disk 353.0 MB, gpus: 0.0, runtime 1086.516848 processing task id 675 item 41a614f5 with 111634 events completed on d12chas553.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.221, memory: 2361.0 MB, disk 348.0 MB, gpus: 0.0, runtime 990.616866 processing task id 583 item 42b3e2b8 with 119233 events completed on d12chas589.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.211, memory: 1933.0 MB, disk 353.0 MB, gpus: 0.0, runtime 1077.418528 processing task id 582 item 9d220c0e with 119248 events completed on d12chas589.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.205, memory: 1237.0 MB, disk 353.0 MB, gpus: 0.0, runtime 1079.201778 processing task id 706 item f5d0969e with 114750 events completed on d12chas553.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.211, memory: 2244.0 MB, disk 357.0 MB, gpus: 0.0, runtime 926.204872 accumulating task id 809 item accum_13 with 31010923 events completed on d12chas553.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 3750.0 MB, disk: 2000.0 MB, gpus: 0.0 measured cores: 0.998, memory: 3160.0 MB, disk 1655.0 MB, gpus: 0.0, runtime 26.462373 processing task id 564 item c2ca64a7 with 119156 events completed on d12chas577.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.216, memory: 1231.0 MB, disk 353.0 MB, gpus: 0.0, runtime 1092.934918 processing task id 717 item bee9251a with 84000 events completed on d12chas547.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.189, memory: 1632.0 MB, disk 354.0 MB, gpus: 0.0, runtime 878.326386 processing task id 753 item f098e7f0 with 86500 events completed on qa-rtx6k-021.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.142, memory: 892.0 MB, disk 334.0 MB, gpus: 0.0, runtime 684.138221 processing task id 616 item f765e3fa with 117615 events completed on d12chas584.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.247, memory: 2291.0 MB, disk 353.0 MB, gpus: 0.0, runtime 1095.869188 processing task id 688 item 353e4b0a with 117970 events completed on qa-rtx6k-021.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.513, memory: 1221.0 MB, disk 348.0 MB, gpus: 0.0, runtime 104.85408 processing task id 757 item d5b80e41 with 87500 events completed on d12chas587.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.195, memory: 957.0 MB, disk 334.0 MB, gpus: 0.0, runtime 728.713536 processing task id 744 item 9d9cc3b8 with 84334 events completed on d12chas547.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.205, memory: 965.0 MB, disk 354.0 MB, gpus: 0.0, runtime 744.588652 processing task id 755 item 927b92f7 with 86500 events completed on d12chas580.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.2, memory: 946.0 MB, disk 334.0 MB, gpus: 0.0, runtime 728.067004 processing task id 807 item 943290a8 with 114500 events completed on d12chas553.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 3250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.358, memory: 2339.0 MB, disk 357.0 MB, gpus: 0.0, runtime 350.065005 processing task id 806 item 63afbf13 with 123250 events completed on d12chas582.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 3250.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.363, memory: 1271.0 MB, disk 358.0 MB, gpus: 0.0, runtime 363.79355 processing task id 584 item ba606236 with 119522 events completed on qa-rtx6k-021.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.676, memory: 1230.0 MB, disk 353.0 MB, gpus: 0.0, runtime 84.260771 processing task id 798 item d8cf3acf with 83834 events completed on d12chas546.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2750.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.189, memory: 927.0 MB, disk 334.0 MB, gpus: 0.0, runtime 742.630558 submitted accumulation task id 810 item accum_14, with 33595786 events processing task id 760 item 7b4ae8c3 with 87834 events completed on d12chas551.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.174, memory: 956.0 MB, disk 334.0 MB, gpus: 0.0, runtime 789.245597 processing task id 754 item 60604d06 with 86500 events completed on d12chas546.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.184, memory: 1240.0 MB, disk 334.0 MB, gpus: 0.0, runtime 787.634009 accumulating task id 810 item accum_14 with 33595786 events completed on d12chas582.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 3750.0 MB, disk: 2000.0 MB, gpus: 0.0 measured cores: 0.998, memory: 3038.0 MB, disk 1569.0 MB, gpus: 0.0, runtime 24.885002 processing task id 776 item a4f14797 with 86167 events completed on d12chas547.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2750.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.184, memory: 1884.0 MB, disk 334.0 MB, gpus: 0.0, runtime 820.223216 processing task id 775 item 16bb2041 with 86167 events completed on d12chas589.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2750.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.189, memory: 952.0 MB, disk 335.0 MB, gpus: 0.0, runtime 823.207536 processing task id 800 item 11a1f651 with 83832 events completed on d12chas584.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2750.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.189, memory: 1104.0 MB, disk 334.0 MB, gpus: 0.0, runtime 817.342911 processing task id 791 item 8a4d5aa8 with 119250 events completed on d12chas584.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2750.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.226, memory: 1185.0 MB, disk 335.0 MB, gpus: 0.0, runtime 819.626099 processing task id 555 item ad705893 with 115080 events completed on d12chas580.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.258, memory: 2527.0 MB, disk 353.0 MB, gpus: 0.0, runtime 826.411615 processing task id 784 item e1a7b8b6 with 87500 events completed on d12chas552.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2750.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.189, memory: 1757.0 MB, disk 334.0 MB, gpus: 0.0, runtime 827.000117 processing task id 786 item c54dc74b with 120000 events completed on d12chas577.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2750.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.226, memory: 1171.0 MB, disk 335.0 MB, gpus: 0.0, runtime 829.282431 processing task id 484 item d628953f with 115140 events completed on d12chas564.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.263, memory: 2257.0 MB, disk 349.0 MB, gpus: 0.0, runtime 878.31057 processing task id 801 item 16617b8b with 118750 events completed on d12chas584.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2750.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.242, memory: 1179.0 MB, disk 335.0 MB, gpus: 0.0, runtime 837.006334 processing task id 640 item 6b31a190 with 117814 events completed on d12chas553.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.221, memory: 2439.0 MB, disk 364.0 MB, gpus: 0.0, runtime 1278.921941 processing task id 785 item 0c7f8184 with 87500 events completed on d12chas552.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2750.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.205, memory: 956.0 MB, disk 334.0 MB, gpus: 0.0, runtime 844.950535 processing task id 729 item bfa76817 with 94500 events completed on d12chas580.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.205, memory: 1170.0 MB, disk 354.0 MB, gpus: 0.0, runtime 1013.682721 processing task id 613 item 909389a4 with 107933 events completed on d12chas546.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.232, memory: 2170.0 MB, disk 352.0 MB, gpus: 0.0, runtime 1288.993735 processing task id 648 item e1aa23c1 with 118915 events completed on d12chas551.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.221, memory: 1219.0 MB, disk 365.0 MB, gpus: 0.0, runtime 1283.275202 processing task id 660 item afeb9b08 with 109534 events completed on d12chas546.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.237, memory: 1156.0 MB, disk 348.0 MB, gpus: 0.0, runtime 1273.569246 processing task id 799 item 6b40cd9f with 83834 events completed on d12chas546.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2750.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.189, memory: 925.0 MB, disk 334.0 MB, gpus: 0.0, runtime 869.354561 processing task id 685 item 57fb30c9 with 111196 events completed on qa-rtx6k-021.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.184, memory: 1169.0 MB, disk 348.0 MB, gpus: 0.0, runtime 1205.39637 processing task id 670 item 76d4213f with 118117 events completed on d12chas552.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.242, memory: 2452.0 MB, disk 349.0 MB, gpus: 0.0, runtime 1253.604528 processing task id 466 item bcea9e95 with 109535 events completed on d12chas564.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.247, memory: 1156.0 MB, disk 348.0 MB, gpus: 0.0, runtime 929.351283 processing task id 750 item 8a70bd18 with 119000 events completed on d12chas580.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2750.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.232, memory: 2501.0 MB, disk 335.0 MB, gpus: 0.0, runtime 888.11956 processing task id 507 item 450d61a2 with 112585 events completed on d12chas580.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.232, memory: 2172.0 MB, disk 348.0 MB, gpus: 0.0, runtime 950.826492 processing task id 506 item 25cfe105 with 111195 events completed on d12chas564.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.226, memory: 1665.0 MB, disk 348.0 MB, gpus: 0.0, runtime 951.644967 submitted accumulation task id 811 item accum_15, with 36103664 events processing task id 558 item b9125432 with 118381 events completed on d12chas552.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.253, memory: 1231.0 MB, disk 353.0 MB, gpus: 0.0, runtime 1361.326373 processing task id 709 item fb4260f6 with 123250 events completed on d12chas553.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.247, memory: 1263.0 MB, disk 358.0 MB, gpus: 0.0, runtime 1197.079724 processing task id 745 item d7a8bffd with 84334 events completed on d12chas564.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.205, memory: 1938.0 MB, disk 354.0 MB, gpus: 0.0, runtime 958.405841 processing task id 704 item 2384d4e3 with 115750 events completed on d12chas552.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.242, memory: 1204.0 MB, disk 357.0 MB, gpus: 0.0, runtime 1217.131032 processing task id 708 item 96480b02 with 114750 events completed on qa-rtx6k-021.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.253, memory: 1629.0 MB, disk 357.0 MB, gpus: 0.0, runtime 289.392115 accumulating task id 811 item accum_15 with 36103664 events completed on d12chas582.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 3750.0 MB, disk: 2000.0 MB, gpus: 0.0 measured cores: 0.997, memory: 3124.0 MB, disk 1426.0 MB, gpus: 0.0, runtime 33.395484 processing task id 565 item 312973ef with 119598 events completed on d12chas553.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.305, memory: 2567.0 MB, disk 353.0 MB, gpus: 0.0, runtime 333.934939 processing task id 707 item 2933c61b with 115750 events completed on d12chas587.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.247, memory: 1205.0 MB, disk 357.0 MB, gpus: 0.0, runtime 1247.070209 processing task id 635 item 937a33eb with 118057 events completed on d12chas564.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 2500.0 MB, disk: 500.0 MB, gpus: 0.0 measured cores: 0.258, memory: 1232.0 MB, disk 349.0 MB, gpus: 0.0, runtime 1437.283059 processing task id 699 item bfac42b9 with 122000 events completed on qa-rtx6k-021.crc.nd.edu. return code 0 allocated cores: 1.0, memory: 10000.0 MB, disk: 8000.0 MB, gpus: 0.0 measured cores: 0.3, memory: 2642.0 MB, disk 358.0 MB, gpus: 0.0, runtime 248.847669 Submitted 100%|███████████████████████████████████████████████████████| 41319784/41319784 [31:48<00:00, 120243.14event/s] ^******************************************************************************** Canceling processing tasks for final accumulation. C-c again to immediately terminate. ******************************************************************************** Submitted 100%|███████████████████████████████████████████████████████| 41319784/41319784 [51:08<00:00, 13466.18event/s] Processing 90%|█████████████████████████████████████████████████▍ | 37135534/41319784 [51:08<05:45, 12102.52event/s] Accumulated 88%|████████████████████████████████████████████████▌ | 15/17 [51:08<06:49, 204.56s/tasks] WARNING: Not all items were processed. Not all results (10) were accumulated in an accumulation job. Accumulating locally. Performing final accumulation... Processed 41319784 events in 3387.96915602684 seconds (12196.03 evts/sec). Filled 110839428 bins, nonzero bins: 24.1 % Saving output in histos/output_check_yields.pkl.gz... Done! real 60m12.709s user 2m42.752s sys 4m12.306s