From 48fdb05a0cb0e551846460996ef2a74fe55d185f Mon Sep 17 00:00:00 2001 From: Alan Malta Rodrigues Date: Wed, 31 Jul 2024 11:33:21 -0400 Subject: [PATCH] fix unit tests for RucioConMon temporarily remove integration tag for unit tests fix RucioConMon unit test fix MSUnmerged unit tests resolve MSUnmerged unit tests --- .../MSUnmerged_t/MSUnmerged_t.py | 73 ++++++++----------- .../Services_t/Rucio_t/RucioConMon_t.py | 11 ++- 2 files changed, 37 insertions(+), 47 deletions(-) diff --git a/test/python/WMCore_t/MicroService_t/MSUnmerged_t/MSUnmerged_t.py b/test/python/WMCore_t/MicroService_t/MSUnmerged_t/MSUnmerged_t.py index 33a11b093e..4be2e86d57 100644 --- a/test/python/WMCore_t/MicroService_t/MSUnmerged_t/MSUnmerged_t.py +++ b/test/python/WMCore_t/MicroService_t/MSUnmerged_t/MSUnmerged_t.py @@ -8,7 +8,6 @@ import os import unittest -from future.utils import viewkeys from mock import mock from Utils.PythonVersion import PY3 @@ -70,8 +69,8 @@ def getBasicRSEData(): rse = {"name": "T2_TestRSE", "counters": {"dirsToDeleteAll": 0}, "dirs": {"allUnmerged": set(), - "toDelete": {}, - "protected": []}, + "toDelete": set(), + "protected": set()}, "files": {"allUnmerged": set(lfns), "toDelete": {}, "protected": []} @@ -168,13 +167,13 @@ def testPlineUnmerged(self): rse = self.msUnmerged.updateRSETimestamps(rse, start=True, end=False) rse = self.msUnmerged.consRecordAge(rse) rse = self.msUnmerged.getUnmergedFiles(rse) - rse = self.msUnmerged.filterUnmergedFiles(rse) + rse = self.msUnmerged.getPfn(rse) rse = self.msUnmerged.cleanRSE(rse) rse = self.msUnmerged.updateServiceCounters(rse) rse = self.msUnmerged.updateRSETimestamps(rse, start=False, end=True) # self.msUnmerged.plineUnmerged.run(rse) expectedRSE = {'name': 'T2_US_Wisconsin', - 'pfnPrefix': None, + 'pfnPrefix': mock.ANY, 'isClean': False, 'rucioConMonStatus': None, 'timestamps': {'endTime': mock.ANY, @@ -185,23 +184,18 @@ def testPlineUnmerged(self): "counters": {"totalNumFiles": 11938, "totalNumDirs": 11, "dirsToDelete": 6, - "filesToDelete": 0, + "filesToDelete": 10934, "filesDeletedSuccess": 0, "filesDeletedFail": 0, "dirsDeletedSuccess": 0, "dirsDeletedFail": 0, "gfalErrors": {}}, - 'files': {'allUnmerged': mock.ANY, + 'files': {'allUnmerged': [], 'deletedFail': set(), 'deletedSuccess': set(), 'protected': {}, - 'toDelete': {'/store/unmerged/Phase2HLTTDRSummer20ReRECOMiniAOD/DYToLL_M-50_TuneCP5_14TeV-pythia8/FEVT/FlatPU0To200_pilot_111X_mcRun4_realistic_T15_v1-v2': mock.ANY, - '/store/unmerged/Run2016G/DoubleEG/MINIAOD/UL2016_MiniAODv2-v1': mock.ANY, - '/store/unmerged/SAM/testSRM/SAM-cms-lvs-gridftp.hep.wisc.edu': mock.ANY, - '/store/unmerged/SAM/testSRM/SAM-cms-lvs-gridftp.hep.wisc.edu/lcg-util': mock.ANY, - '/store/unmerged/SAM/testSRM/SAM-cmssrm.hep.wisc.edu': mock.ANY, - '/store/unmerged/SAM/testSRM/SAM-cmssrm.hep.wisc.edu/lcg-util': mock.ANY}}, - 'dirs': {'allUnmerged': set(), + 'toDelete': {}}, + 'dirs': {'allUnmerged': [], "deletedSuccess": set(), "deletedFail": set(), 'protected': {'/store/unmerged/RunIIAutumn18FSPremix/PMSSM_set_1_prompt_1_TuneCP2_13TeV-pythia8/AODSIM/GridpackScan_102X_upgrade2018_realistic_v15-v1', @@ -233,45 +227,40 @@ def testCutPath(self): def testFilterInclDirectories(self): "Test MSUnmerged with including directories filter" - toDeleteDict = {"/store/unmerged/data/prod/2018/1/12": ["/store/unmerged/data/prod/2018/1/12/log6.tar"], - "/store/unmerged/express/prod/2020/1/12": ["/store/unmerged/express/prod/2020/1/12/log8.tar", - "/store/unmerged/express/prod/2020/1/12/log9.tar"]} + toDeleteDict = {"/store/unmerged/data/prod/2018/1/12", "/store/unmerged/express/prod/2020/1/12"} rseData = getBasicRSEData() self.msUnmerged.msConfig['dirFilterIncl'] = ["/store/unmerged/data/prod/2018/", "/store/unmerged/express"] self.msUnmerged.protectedLFNs = set() - filterData = self.msUnmerged.filterUnmergedFiles(rseData) - self.assertEqual(filterData['counters']['dirsToDelete'], 2) - self.assertItemsEqual(viewkeys(filterData['files']['toDelete']), viewkeys(toDeleteDict)) - self.assertItemsEqual(list(filterData['files']['toDelete']['/store/unmerged/data/prod/2018/1/12']), - toDeleteDict['/store/unmerged/data/prod/2018/1/12']) - self.assertItemsEqual(list(filterData['files']['toDelete']['/store/unmerged/express/prod/2020/1/12']), - toDeleteDict['/store/unmerged/express/prod/2020/1/12']) + filterData = set() + for dirPath in rseData['dirs']['allUnmerged']: + if self.msUnmerged._isDeletable(dirPath): + filterData.add(dirPath) + + self.assertEqual(len(filterData), 2) + self.assertItemsEqual(filterData, toDeleteDict) def testFilterExclDirectories(self): "Test MSUnmerged with excluding directories filter" - toDeleteDict = {"/store/unmerged/data/prod/2018/1/12": ["/store/unmerged/data/prod/2018/1/12/log6.tar"], - "/store/unmerged/express/prod/2020/1/12": ["/store/unmerged/express/prod/2020/1/12/log8.tar", - "/store/unmerged/express/prod/2020/1/12/log9.tar"]} + toDeleteDict = {"/store/unmerged/data/prod/2018/1/12", "/store/unmerged/express/prod/2020/1/12"} rseData = getBasicRSEData() self.msUnmerged.msConfig['dirFilterExcl'] = ["/store/unmerged/logs", "/store/unmerged/data/prod/2019", "/store/unmerged/alan/prod"] self.msUnmerged.protectedLFNs = set() - filterData = self.msUnmerged.filterUnmergedFiles(rseData) - self.assertEqual(filterData['counters']['dirsToDelete'], 2) - self.assertItemsEqual(viewkeys(filterData['files']['toDelete']), viewkeys(toDeleteDict)) - self.assertItemsEqual(list(filterData['files']['toDelete']['/store/unmerged/data/prod/2018/1/12']), - toDeleteDict['/store/unmerged/data/prod/2018/1/12']) - self.assertItemsEqual(list(filterData['files']['toDelete']['/store/unmerged/express/prod/2020/1/12']), - toDeleteDict['/store/unmerged/express/prod/2020/1/12']) + filterData = set() + for dirPath in rseData['dirs']['allUnmerged']: + if self.msUnmerged._isDeletable(dirPath): + filterData.add(dirPath) + + self.assertEqual(len(filterData), 2) + self.assertItemsEqual(filterData, toDeleteDict) def testFilterInclExclDirectories(self): "Test MSUnmerged with including and excluding directories filter" - toDeleteDict = {"/store/unmerged/express/prod/2020/1/12": ["/store/unmerged/express/prod/2020/1/12/log8.tar", - "/store/unmerged/express/prod/2020/1/12/log9.tar"]} + toDeleteDict = {"/store/unmerged/express/prod/2020/1/12"} rseData = getBasicRSEData() self.msUnmerged.msConfig['dirFilterIncl'] = ["/store/unmerged/data/prod/2018/", "/store/unmerged/express"] @@ -279,8 +268,10 @@ def testFilterInclExclDirectories(self): "/store/unmerged/data/prod", "/store/unmerged/alan/prod"] self.msUnmerged.protectedLFNs = set() - filterData = self.msUnmerged.filterUnmergedFiles(rseData) - self.assertEqual(filterData['counters']['dirsToDelete'], 1) - self.assertItemsEqual(viewkeys(filterData['files']['toDelete']), viewkeys(toDeleteDict)) - self.assertItemsEqual(list(filterData['files']['toDelete']['/store/unmerged/express/prod/2020/1/12']), - toDeleteDict['/store/unmerged/express/prod/2020/1/12']) + filterData = set() + for dirPath in rseData['dirs']['allUnmerged']: + if self.msUnmerged._isDeletable(dirPath): + filterData.add(dirPath) + + self.assertEqual(len(filterData), 1) + self.assertItemsEqual(filterData, toDeleteDict) diff --git a/test/python/WMCore_t/Services_t/Rucio_t/RucioConMon_t.py b/test/python/WMCore_t/Services_t/Rucio_t/RucioConMon_t.py index 77343c42f6..98607cf599 100644 --- a/test/python/WMCore_t/Services_t/Rucio_t/RucioConMon_t.py +++ b/test/python/WMCore_t/Services_t/Rucio_t/RucioConMon_t.py @@ -8,7 +8,6 @@ import unittest from nose.plugins.attrib import attr - from WMCore.Services.RucioConMon.RucioConMon import RucioConMon @@ -17,17 +16,17 @@ class RucioConMonTest(unittest.TestCase): Unit tests for RucioConMon Service module """ - @attr("integration") +# @attr("integration") def testGetRSEUnmerged(self): """ Test getRSEUnmerged method using both zipped and unzipped requests This test uses specific rse name which can be changed to any other RSE. """ # url = "https://cmsweb.cern.ch/rucioconmon/WM/files?rse=T2_TR_METU&format=raw" - mgr = RucioConMon("https://cmsweb.cern.ch/rucioconmon") - rseName = "T2_TR_METU" - dataUnzipped = mgr.getRSEUnmerged(rseName, zipped=False) - dataZipped = mgr.getRSEUnmerged(rseName, zipped=True) + mgr = RucioConMon("https://cmsweb.cern.ch/rucioconmon/unmerged") + rseName = "T2_RU_ITEP" + dataUnzipped = [item for item in mgr.getRSEUnmerged(rseName, zipped=False)] + dataZipped = [item for item in mgr.getRSEUnmerged(rseName, zipped=True)] self.assertTrue(dataUnzipped == dataZipped)