From 8b9a693f9bc4030eaa3fbcaa394bb7af74ed10bf Mon Sep 17 00:00:00 2001 From: John Stilley <1831479+john-science@users.noreply.github.com> Date: Tue, 26 Nov 2024 15:50:09 -0800 Subject: [PATCH] Moving from shutil.copy to safeCopy (#2024) --- armi/bookkeeping/db/database.py | 5 ++- armi/bookkeeping/db/tests/test_database3.py | 16 ++++----- armi/bookkeeping/report/newReports.py | 18 +++++----- armi/operators/operator.py | 34 ++++++++----------- .../neutronics/crossSectionGroupManager.py | 8 ++--- .../latticePhysics/latticePhysicsInterface.py | 21 +++++------- armi/settings/settingsValidation.py | 15 +++----- armi/utils/directoryChangers.py | 7 ++-- armi/utils/outputCache.py | 9 +++-- armi/utils/pathTools.py | 8 ++--- doc/conf.py | 3 +- 11 files changed, 62 insertions(+), 82 deletions(-) diff --git a/armi/bookkeeping/db/database.py b/armi/bookkeeping/db/database.py index 9a067581e..5a718cda8 100644 --- a/armi/bookkeeping/db/database.py +++ b/armi/bookkeeping/db/database.py @@ -77,8 +77,7 @@ from armi.reactor.reactorParameters import makeParametersReadOnly from armi.reactor.reactors import Core, Reactor from armi.settings.fwSettings.globalSettings import CONF_SORT_REACTOR -from armi.utils import getNodesPerCycle -from armi.utils import safeMove +from armi.utils import getNodesPerCycle, safeCopy, safeMove from armi.utils.textProcessors import resolveMarkupInclusions # CONSTANTS @@ -700,7 +699,7 @@ def syncToSharedFolder(self): # Close the h5 file so it can be copied self.h5db.close() self.h5db = None - shutil.copy(self._fullPath, self._fileName) + safeCopy(self._fullPath, self._fileName) # Garbage collect so we don't have multiple databases hanging around in memory gc.collect() diff --git a/armi/bookkeeping/db/tests/test_database3.py b/armi/bookkeeping/db/tests/test_database3.py index 946078337..c4e769ea9 100644 --- a/armi/bookkeeping/db/tests/test_database3.py +++ b/armi/bookkeeping/db/tests/test_database3.py @@ -12,31 +12,27 @@ # See the License for the specific language governing permissions and # limitations under the License. """Tests for the Database class.""" -from glob import glob import io import os import shutil import subprocess import unittest +from glob import glob import h5py import numpy as np -from armi.bookkeeping.db import _getH5File -from armi.bookkeeping.db import database -from armi.bookkeeping.db import loadOperator +from armi.bookkeeping.db import _getH5File, database, loadOperator from armi.bookkeeping.db.databaseInterface import DatabaseInterface from armi.bookkeeping.db.jaggedArray import JaggedArray from armi.reactor import parameters from armi.reactor.excoreStructure import ExcoreCollection, ExcoreStructure -from armi.reactor.reactors import Core -from armi.reactor.reactors import Reactor +from armi.reactor.reactors import Core, Reactor from armi.reactor.spentFuelPool import SpentFuelPool from armi.reactor.tests.test_reactors import loadTestReactor, reduceTestReactorRings from armi.settings.fwSettings.globalSettings import CONF_SORT_REACTOR -from armi.tests import mockRunLogs -from armi.tests import TEST_ROOT -from armi.utils import getPreviousTimeNode +from armi.tests import TEST_ROOT, mockRunLogs +from armi.utils import getPreviousTimeNode, safeCopy from armi.utils.directoryChangers import TemporaryDirectoryChanger # determine if this is a parallel run, and git is installed @@ -783,7 +779,7 @@ def setUp(self): thisDir = self.td.destination yamls = glob(os.path.join(TEST_ROOT, "smallestTestReactor", "*.yaml")) for yam in yamls: - shutil.copy(os.path.join(TEST_ROOT, "smallestTestReactor", yam), thisDir) + safeCopy(os.path.join(TEST_ROOT, "smallestTestReactor", yam), thisDir) # Add an EVST to this reactor with open("refSmallestReactor.yaml", "w") as f: diff --git a/armi/bookkeeping/report/newReports.py b/armi/bookkeeping/report/newReports.py index ab6f214f1..d44848207 100644 --- a/armi/bookkeeping/report/newReports.py +++ b/armi/bookkeeping/report/newReports.py @@ -12,22 +12,20 @@ # See the License for the specific language governing permissions and # limitations under the License. -from abc import ABC, abstractmethod -from enum import Enum -from enum import auto -from operator import itemgetter -from typing import Union, Dict import base64 import collections import copy import os -import shutil +from abc import ABC, abstractmethod +from enum import Enum, auto +from operator import itemgetter +from typing import Dict, Union import htmltree import matplotlib.pyplot as plt -from armi import context -from armi import runLog +from armi import context, runLog +from armi.utils import safeCopy class ReportContent: @@ -86,14 +84,14 @@ def writeReports(self): doc = htmltree.Html(head, body) # Copy css file to the correct folder containing the reportContent.html - shutil.copy( + safeCopy( os.path.abspath( os.path.join(os.path.abspath(__file__), os.pardir, "styles.css") ), "styles.css", ) - shutil.copy( + safeCopy( os.path.abspath( os.path.join(os.path.abspath(__file__), os.pardir, "report.js") ), diff --git a/armi/operators/operator.py b/armi/operators/operator.py index 09df06b3a..c3cf4eb95 100644 --- a/armi/operators/operator.py +++ b/armi/operators/operator.py @@ -25,13 +25,10 @@ import collections import os import re -import shutil import time from typing import Tuple -from armi import context -from armi import interfaces -from armi import runLog +from armi import context, interfaces, runLog from armi.bookkeeping import memoryProfiler from armi.bookkeeping.report import reportingUtils from armi.operators.runTypes import RunTypes @@ -41,22 +38,22 @@ ) from armi.settings import settingsValidation from armi.settings.fwSettings.globalSettings import ( - CONF_TIGHT_COUPLING, - CONF_TIGHT_COUPLING_MAX_ITERS, CONF_CYCLES_SKIP_TIGHT_COUPLING_INTERACTION, CONF_DEFERRED_INTERFACE_NAMES, CONF_DEFERRED_INTERFACES_CYCLE, + CONF_TIGHT_COUPLING, + CONF_TIGHT_COUPLING_MAX_ITERS, ) -from armi.utils import codeTiming from armi.utils import ( - pathTools, - getPowerFractions, + codeTiming, getAvailabilityFactors, - getStepLengths, - getCycleLengths, getBurnSteps, - getMaxBurnSteps, + getCycleLengths, getCycleNames, + getMaxBurnSteps, + getPowerFractions, + getStepLengths, + pathTools, ) @@ -1219,16 +1216,15 @@ def snapshotRequest(self, cycle, node, iteration=None): ) else: newFile = "{0}_{1:03d}_{2:d}{3}".format(base, cycle, node, ext) - # add the cycle and timenode to the XS input file names so that a rx-coeff case that runs - # in here won't overwrite them. - shutil.copy(fileName, os.path.join(newFolder, newFile)) + # add the cycle and timenode to the XS input file names so that a rx-coeff case that + # runs in here won't overwrite them. + pathTools.copyOrWarn( + fileName, fileName, os.path.join(newFolder, newFile) + ) if "rzmflx" in fileName: pathTools.copyOrWarn("rzmflx for snapshot", fileName, newFolder) - fileNamePossibilities = [ - f"ISOTXS-c{cycle}n{node}", - f"ISOTXS-c{cycle}", - ] + fileNamePossibilities = [f"ISOTXS-c{cycle}n{node}", f"ISOTXS-c{cycle}"] if iteration is not None: fileNamePossibilities = [ f"ISOTXS-c{cycle}n{node}i{iteration}" diff --git a/armi/physics/neutronics/crossSectionGroupManager.py b/armi/physics/neutronics/crossSectionGroupManager.py index 36022c576..1dac6f13a 100644 --- a/armi/physics/neutronics/crossSectionGroupManager.py +++ b/armi/physics/neutronics/crossSectionGroupManager.py @@ -54,7 +54,6 @@ import collections import copy import os -import shutil import string import numpy as np @@ -66,7 +65,8 @@ from armi.reactor.components import basicShapes from armi.reactor.converters.blockConverters import stripComponents from armi.reactor.flags import Flags -from armi.utils.units import TRACE_NUMBER_DENSITY, C_TO_K +from armi.utils import safeCopy +from armi.utils.units import C_TO_K, TRACE_NUMBER_DENSITY ORDER = interfaces.STACK_ORDER.BEFORE + interfaces.STACK_ORDER.CROSS_SECTIONS @@ -1195,7 +1195,7 @@ def _copyPregeneratedXSFile(self, xsID): ) # Prevent copy error if the path and destination are the same. if xsFileLocation != dest: - shutil.copy(xsFileLocation, dest) + safeCopy(xsFileLocation, dest) def _copyPregeneratedFluxSolutionFile(self, xsID): # stop a race condition to copy files between all processors @@ -1211,7 +1211,7 @@ def _copyPregeneratedFluxSolutionFile(self, xsID): ) # Prevent copy error if the path and destination are the same. if fluxFileLocation != dest: - shutil.copy(fluxFileLocation, dest) + safeCopy(fluxFileLocation, dest) def _getPregeneratedXsFileLocationData(self, xsID): """ diff --git a/armi/physics/neutronics/latticePhysics/latticePhysicsInterface.py b/armi/physics/neutronics/latticePhysics/latticePhysicsInterface.py index 0a1c4cc9c..beb5aaa79 100644 --- a/armi/physics/neutronics/latticePhysics/latticePhysicsInterface.py +++ b/armi/physics/neutronics/latticePhysics/latticePhysicsInterface.py @@ -18,22 +18,19 @@ Parent classes for codes responsible for generating broad-group cross sections """ import os -import shutil -from armi import interfaces -from armi import nuclearDataIO -from armi import runLog +from armi import interfaces, nuclearDataIO, runLog from armi.physics import neutronics +from armi.physics.neutronics import LatticePhysicsFrequency from armi.physics.neutronics.const import CONF_CROSS_SECTION from armi.physics.neutronics.settings import ( - CONF_GEN_XS, CONF_CLEAR_XS, + CONF_GEN_XS, + CONF_LATTICE_PHYSICS_FREQUENCY, CONF_TOLERATE_BURNUP_CHANGE, CONF_XS_KERNEL, - CONF_LATTICE_PHYSICS_FREQUENCY, ) -from armi.physics.neutronics import LatticePhysicsFrequency -from armi.utils import codeTiming +from armi.utils import codeTiming, safeCopy LATTICE_PHYSICS = "latticePhysics" @@ -149,17 +146,17 @@ def updateXSLibrary(self, cycle, node=None): def _renameExistingLibrariesForStatepoint(self, cycle, node): """Copy the existing neutron and/or gamma libraries into cycle-dependent files.""" - shutil.copy( + safeCopy( neutronics.ISOTXS, nuclearDataIO.getExpectedISOTXSFileName(cycle, node) ) if self.includeGammaXS: - shutil.copy( + safeCopy( neutronics.GAMISO, nuclearDataIO.getExpectedGAMISOFileName( cycle=cycle, node=node, suffix=self._getSuffix(cycle) ), ) - shutil.copy( + safeCopy( neutronics.PMATRX, nuclearDataIO.getExpectedPMATRXFileName( cycle=cycle, node=node, suffix=self._getSuffix(cycle) @@ -195,7 +192,7 @@ def _copyLibraryFilesForCycle(cycle, libFiles): else: runLog.info("Using {} as an active library".format(baseName)) if cycleName != baseName: - shutil.copy(cycleName, baseName) + safeCopy(cycleName, baseName) def _readGammaBinaries(self, lib, gamisoFileName, pmatrxFileName): raise NotImplementedError( diff --git a/armi/settings/settingsValidation.py b/armi/settings/settingsValidation.py index 6a4534a79..f483843ef 100644 --- a/armi/settings/settingsValidation.py +++ b/armi/settings/settingsValidation.py @@ -24,21 +24,16 @@ import itertools import os import re -import shutil -from armi import context -from armi import getPluginManagerOrFail -from armi import runLog +from armi import context, getPluginManagerOrFail, runLog from armi.physics import neutronics -from armi.reactor import geometry -from armi.reactor import systemLayoutInput +from armi.reactor import geometry, systemLayoutInput from armi.settings.settingsIO import ( - prompt, RunLogPromptCancel, RunLogPromptUnresolvable, + prompt, ) -from armi.utils import directoryChangers -from armi.utils import pathTools +from armi.utils import directoryChangers, pathTools, safeCopy from armi.utils.mathematics import expandRepeatedFloats @@ -245,7 +240,7 @@ def run(self, cs=None): runLog.important( f"Preserving original settings file by renaming `{renamePath}`" ) - shutil.copy(self.cs.path, renamePath) + safeCopy(self.cs.path, renamePath) # save settings file self.cs.writeToYamlFile(self.cs.path) diff --git a/armi/utils/directoryChangers.py b/armi/utils/directoryChangers.py index 773452d5d..50a5c0cbd 100644 --- a/armi/utils/directoryChangers.py +++ b/armi/utils/directoryChangers.py @@ -22,7 +22,7 @@ from armi import context from armi import runLog from armi.utils import pathTools -from armi.utils import safeMove +from armi.utils import safeCopy, safeMove def _changeDirectory(destination): @@ -187,8 +187,7 @@ def _transferFiles(initialPath, destinationPath, fileList, moveFiles=False): Transfer files into or out of the directory. This is used in ``moveFiles`` and ``retrieveFiles`` to shuffle files about when creating a - target directory or when coming back, respectively. Beware that this uses ``shutil.copy()`` - under the hood, which doesn't play nicely with directories. + target directory or when coming back, respectively. Parameters ---------- @@ -239,7 +238,7 @@ def _transferFiles(initialPath, destinationPath, fileList, moveFiles=False): safeMove(fromPath, toPath) else: runLog.extra("Copying {} to {}".format(fromPath, toPath)) - shutil.copy(fromPath, toPath) + safeCopy(fromPath, toPath) class TemporaryDirectoryChanger(DirectoryChanger): diff --git a/armi/utils/outputCache.py b/armi/utils/outputCache.py index e9f4b0ebc..1d0ae750d 100644 --- a/armi/utils/outputCache.py +++ b/armi/utils/outputCache.py @@ -36,17 +36,16 @@ Notes ----- -Could probably be, like, a decorate on subprocess but we call subprocess a bunch of -different ways. +Could probably be, like, a decorate on subprocess but we call subprocess a bunch of different ways. """ import hashlib import json import os -import shutil import subprocess from armi import runLog +from armi.utils import safeCopy from armi.utils.pathTools import cleanPath MANIFEST_NAME = "CRC-manifest.json" @@ -106,7 +105,7 @@ def _copyOutputs(cachedFolder, locToRetrieveTo): for copy in copies: storedOutputPath, copyPath = copy - shutil.copy(storedOutputPath, copyPath) + safeCopy(storedOutputPath, copyPath) return True @@ -166,7 +165,7 @@ def store(exePath, inputPaths, outputFiles, cacheDir): for outputFile in outputsThatExist: baseName = os.path.basename(outputFile) cachedLoc = os.path.join(folderLoc, baseName) - shutil.copy(outputFile, cachedLoc) + safeCopy(outputFile, cachedLoc) runLog.info("Added outputs for {} to the cache.".format(exePath)) diff --git a/armi/utils/pathTools.py b/armi/utils/pathTools.py index d80354a09..064bf33c6 100644 --- a/armi/utils/pathTools.py +++ b/armi/utils/pathTools.py @@ -16,14 +16,14 @@ This module contains commonly used functions relating to directories, files and path manipulations. """ -from time import sleep import importlib import os import pathlib import shutil +from time import sleep -from armi import context -from armi import runLog +from armi import context, runLog +from armi.utils import safeCopy DO_NOT_CLEAN_PATHS = [ "armiruns", @@ -57,7 +57,7 @@ def copyOrWarn(filepathDescription, sourcePath, destinationPath): if os.path.isdir(sourcePath): shutil.copytree(sourcePath, destinationPath, dirs_exist_ok=True) else: - shutil.copy(sourcePath, destinationPath) + safeCopy(sourcePath, destinationPath) runLog.debug( "Copied {}: {} -> {}".format( filepathDescription, sourcePath, destinationPath diff --git a/doc/conf.py b/doc/conf.py index e06966421..1dc187fa4 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -54,6 +54,7 @@ from armi import disableFutureConfigures from armi import meta from armi.bookkeeping import tests as bookkeepingTests +from armi.utils import safeCopy context.Mode.setMode(context.Mode.BATCH) @@ -266,7 +267,7 @@ def setup(app): # directory for running the notebooks is the directory of the link itself, so relative paths # don't work. for path in _TUTORIAL_FILES: - shutil.copy(path, dataDir) + safeCopy(path, dataDir) # If extensions (or modules to document with autodoc) are in another directory,