Skip to content

Commit

Permalink
fix unit tests for RucioConMon
Browse files Browse the repository at this point in the history 
temporarily remove integration tag for unit tests

fix RucioConMon unit test

fix MSUnmerged unit tests

resolve MSUnmerged unit tests
  • Loading branch information
@amaltaro
amaltaro committed Oct 2, 2024
1 parent 128da10 commit 48fdb05
Show file tree
Hide file tree
Showing 2 changed files with 37 additions and 47 deletions.
73 changes: 32 additions & 41 deletions test/python/WMCore_t/MicroService_t/MSUnmerged_t/MSUnmerged_t.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@
import os
import unittest

from future.utils import viewkeys
from mock import mock

from Utils.PythonVersion import PY3
Expand Down Expand Up @@ -70,8 +69,8 @@ def getBasicRSEData():
rse = {"name": "T2_TestRSE",
"counters": {"dirsToDeleteAll": 0},
"dirs": {"allUnmerged": set(),
"toDelete": {},
"protected": []},
"toDelete": set(),
"protected": set()},
"files": {"allUnmerged": set(lfns),
"toDelete": {},
"protected": []}
Expand Down Expand Up @@ -168,13 +167,13 @@ def testPlineUnmerged(self):
rse = self.msUnmerged.updateRSETimestamps(rse, start=True, end=False)
rse = self.msUnmerged.consRecordAge(rse)
rse = self.msUnmerged.getUnmergedFiles(rse)
rse = self.msUnmerged.filterUnmergedFiles(rse)
rse = self.msUnmerged.getPfn(rse)
rse = self.msUnmerged.cleanRSE(rse)
rse = self.msUnmerged.updateServiceCounters(rse)
rse = self.msUnmerged.updateRSETimestamps(rse, start=False, end=True)
# self.msUnmerged.plineUnmerged.run(rse)
expectedRSE = {'name': 'T2_US_Wisconsin',
'pfnPrefix': None,
'pfnPrefix': mock.ANY,
'isClean': False,
'rucioConMonStatus': None,
'timestamps': {'endTime': mock.ANY,
Expand All @@ -185,23 +184,18 @@ def testPlineUnmerged(self):
"counters": {"totalNumFiles": 11938,
"totalNumDirs": 11,
"dirsToDelete": 6,
"filesToDelete": 0,
"filesToDelete": 10934,
"filesDeletedSuccess": 0,
"filesDeletedFail": 0,
"dirsDeletedSuccess": 0,
"dirsDeletedFail": 0,
"gfalErrors": {}},
'files': {'allUnmerged': mock.ANY,
'files': {'allUnmerged': [],
'deletedFail': set(),
'deletedSuccess': set(),
'protected': {},
'toDelete': {'/store/unmerged/Phase2HLTTDRSummer20ReRECOMiniAOD/DYToLL_M-50_TuneCP5_14TeV-pythia8/FEVT/FlatPU0To200_pilot_111X_mcRun4_realistic_T15_v1-v2': mock.ANY,
'/store/unmerged/Run2016G/DoubleEG/MINIAOD/UL2016_MiniAODv2-v1': mock.ANY,
'/store/unmerged/SAM/testSRM/SAM-cms-lvs-gridftp.hep.wisc.edu': mock.ANY,
'/store/unmerged/SAM/testSRM/SAM-cms-lvs-gridftp.hep.wisc.edu/lcg-util': mock.ANY,
'/store/unmerged/SAM/testSRM/SAM-cmssrm.hep.wisc.edu': mock.ANY,
'/store/unmerged/SAM/testSRM/SAM-cmssrm.hep.wisc.edu/lcg-util': mock.ANY}},
'dirs': {'allUnmerged': set(),
'toDelete': {}},
'dirs': {'allUnmerged': [],
"deletedSuccess": set(),
"deletedFail": set(),
'protected': {'/store/unmerged/RunIIAutumn18FSPremix/PMSSM_set_1_prompt_1_TuneCP2_13TeV-pythia8/AODSIM/GridpackScan_102X_upgrade2018_realistic_v15-v1',
Expand Down Expand Up @@ -233,54 +227,51 @@ def testCutPath(self):

def testFilterInclDirectories(self):
"Test MSUnmerged with including directories filter"
toDeleteDict = {"/store/unmerged/data/prod/2018/1/12": ["/store/unmerged/data/prod/2018/1/12/log6.tar"],
"/store/unmerged/express/prod/2020/1/12": ["/store/unmerged/express/prod/2020/1/12/log8.tar",
"/store/unmerged/express/prod/2020/1/12/log9.tar"]}
toDeleteDict = {"/store/unmerged/data/prod/2018/1/12", "/store/unmerged/express/prod/2020/1/12"}
rseData = getBasicRSEData()

self.msUnmerged.msConfig['dirFilterIncl'] = ["/store/unmerged/data/prod/2018/",
"/store/unmerged/express"]
self.msUnmerged.protectedLFNs = set()
filterData = self.msUnmerged.filterUnmergedFiles(rseData)
self.assertEqual(filterData['counters']['dirsToDelete'], 2)
self.assertItemsEqual(viewkeys(filterData['files']['toDelete']), viewkeys(toDeleteDict))
self.assertItemsEqual(list(filterData['files']['toDelete']['/store/unmerged/data/prod/2018/1/12']),
toDeleteDict['/store/unmerged/data/prod/2018/1/12'])
self.assertItemsEqual(list(filterData['files']['toDelete']['/store/unmerged/express/prod/2020/1/12']),
toDeleteDict['/store/unmerged/express/prod/2020/1/12'])
filterData = set()
for dirPath in rseData['dirs']['allUnmerged']:
if self.msUnmerged._isDeletable(dirPath):
filterData.add(dirPath)

self.assertEqual(len(filterData), 2)
self.assertItemsEqual(filterData, toDeleteDict)

def testFilterExclDirectories(self):
"Test MSUnmerged with excluding directories filter"
toDeleteDict = {"/store/unmerged/data/prod/2018/1/12": ["/store/unmerged/data/prod/2018/1/12/log6.tar"],
"/store/unmerged/express/prod/2020/1/12": ["/store/unmerged/express/prod/2020/1/12/log8.tar",
"/store/unmerged/express/prod/2020/1/12/log9.tar"]}
toDeleteDict = {"/store/unmerged/data/prod/2018/1/12", "/store/unmerged/express/prod/2020/1/12"}
rseData = getBasicRSEData()

self.msUnmerged.msConfig['dirFilterExcl'] = ["/store/unmerged/logs",
"/store/unmerged/data/prod/2019",
"/store/unmerged/alan/prod"]
self.msUnmerged.protectedLFNs = set()
filterData = self.msUnmerged.filterUnmergedFiles(rseData)
self.assertEqual(filterData['counters']['dirsToDelete'], 2)
self.assertItemsEqual(viewkeys(filterData['files']['toDelete']), viewkeys(toDeleteDict))
self.assertItemsEqual(list(filterData['files']['toDelete']['/store/unmerged/data/prod/2018/1/12']),
toDeleteDict['/store/unmerged/data/prod/2018/1/12'])
self.assertItemsEqual(list(filterData['files']['toDelete']['/store/unmerged/express/prod/2020/1/12']),
toDeleteDict['/store/unmerged/express/prod/2020/1/12'])
filterData = set()
for dirPath in rseData['dirs']['allUnmerged']:
if self.msUnmerged._isDeletable(dirPath):
filterData.add(dirPath)

self.assertEqual(len(filterData), 2)
self.assertItemsEqual(filterData, toDeleteDict)

def testFilterInclExclDirectories(self):
"Test MSUnmerged with including and excluding directories filter"
toDeleteDict = {"/store/unmerged/express/prod/2020/1/12": ["/store/unmerged/express/prod/2020/1/12/log8.tar",
"/store/unmerged/express/prod/2020/1/12/log9.tar"]}
toDeleteDict = {"/store/unmerged/express/prod/2020/1/12"}
rseData = getBasicRSEData()
self.msUnmerged.msConfig['dirFilterIncl'] = ["/store/unmerged/data/prod/2018/",
"/store/unmerged/express"]
self.msUnmerged.msConfig['dirFilterExcl'] = ["/store/unmerged/logs",
"/store/unmerged/data/prod",
"/store/unmerged/alan/prod"]
self.msUnmerged.protectedLFNs = set()
filterData = self.msUnmerged.filterUnmergedFiles(rseData)
self.assertEqual(filterData['counters']['dirsToDelete'], 1)
self.assertItemsEqual(viewkeys(filterData['files']['toDelete']), viewkeys(toDeleteDict))
self.assertItemsEqual(list(filterData['files']['toDelete']['/store/unmerged/express/prod/2020/1/12']),
toDeleteDict['/store/unmerged/express/prod/2020/1/12'])
filterData = set()
for dirPath in rseData['dirs']['allUnmerged']:
if self.msUnmerged._isDeletable(dirPath):
filterData.add(dirPath)

self.assertEqual(len(filterData), 1)
self.assertItemsEqual(filterData, toDeleteDict)
11 changes: 5 additions & 6 deletions test/python/WMCore_t/Services_t/Rucio_t/RucioConMon_t.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@
import unittest

from nose.plugins.attrib import attr

from WMCore.Services.RucioConMon.RucioConMon import RucioConMon


Expand All @@ -17,17 +16,17 @@ class RucioConMonTest(unittest.TestCase):
Unit tests for RucioConMon Service module
"""

@attr("integration")
# @attr("integration")
def testGetRSEUnmerged(self):
"""
Test getRSEUnmerged method using both zipped and unzipped requests
This test uses specific rse name which can be changed to any other RSE.
"""
# url = "https://cmsweb.cern.ch/rucioconmon/WM/files?rse=T2_TR_METU&format=raw"
mgr = RucioConMon("https://cmsweb.cern.ch/rucioconmon")
rseName = "T2_TR_METU"
dataUnzipped = mgr.getRSEUnmerged(rseName, zipped=False)
dataZipped = mgr.getRSEUnmerged(rseName, zipped=True)
mgr = RucioConMon("https://cmsweb.cern.ch/rucioconmon/unmerged")
rseName = "T2_RU_ITEP"
dataUnzipped = [item for item in mgr.getRSEUnmerged(rseName, zipped=False)]
dataZipped = [item for item in mgr.getRSEUnmerged(rseName, zipped=True)]
self.assertTrue(dataUnzipped == dataZipped)


Expand Down

0 comments on commit 48fdb05

Please sign in to comment.