Use pympler to check memory

dmwm · Aug 30, 2024 · f7ec44e · f7ec44e
1 parent 570d1f6
commit f7ec44e
Show file tree

Hide file tree

Showing 2 changed files with 11 additions and 6 deletions.
diff --git a/bin/testRucioConMonMem.py b/bin/testRucioConMonMem.py
@@ -1,7 +1,8 @@
 import os
 import sys
 import logging
-from memory_profiler import profile
+#from memory_profiler import profile
+from pympler import asizeof
 from WMCore.Services.RucioConMon.RucioConMon import RucioConMon
 
 RSE_NAME = "T2_AT_Vienna"
@@ -17,15 +18,16 @@ def loggerSetup(logLevel=logging.INFO):
     return logger
 
 
-profileFp = open('getUnmergedFiles.log', 'w+')
-@profile(stream=profileFp)
+#profileFp = open('getUnmergedFiles.log', 'w+')
+#@profile(stream=profileFp)
 def getUnmergedFiles(rucioConMon, logger, compressed=False):
     dirs = set()
     counter = 0
     logger.info("Fetching data from Rucio ConMon for RSE: %s.", RSE_NAME)
     for lfn in rucioConMon.getRSEUnmerged(RSE_NAME, zipped=compressed):
         dirPath = _cutPath(lfn)
         dirs.add(dirPath)
+        #logger.info(f"Size of dirs object: {asizeof.asizeof(dirs)}")
         counter =+ 1
     logger.info(f"Total files received: {counter}, unique dirs: {len(dirs)}")
     return dirs

diff --git a/src/python/WMCore/Services/RucioConMon/RucioConMon.py b/src/python/WMCore/Services/RucioConMon/RucioConMon.py
@@ -14,7 +14,8 @@
 import gzip
 import json
 import logging
-from memory_profiler import profile
+#from memory_profiler import profile
+from pympler import asizeof
 
 from WMCore.Services.Service import Service
 from Utils.Utilities import decodeBytesToUnicode
@@ -99,8 +100,8 @@ def getRSEStats(self):
         rseStats = self._getResult(uri, callname='stats')
         return rseStats
 
-    profileFp = open('getRSEUnmerged.log', 'w+')
-    @profile(stream=profileFp)
+    #profileFp = open('getRSEUnmerged.log', 'w+')
+    #@profile(stream=profileFp)
     def getRSEUnmerged(self, rseName, zipped=False):
         """
         Gets the list of all unmerged files in an RSE
@@ -122,6 +123,8 @@ def getRSEUnmerged(self, rseName, zipped=False):
             uri = "files?rse=%s&format=json" % rseName
             callname = '{}.json'.format(rseName)
             rseUnmerged = self._getResult(uri, callname=callname)
+
+        self['logger'].info(f"Size of rseUnmerged object: {asizeof.asizeof(rseUnmerged)}")
         # now lazily return items
         for item in rseUnmerged:
             yield item