Turn off debugger hooks in PyTorch?

[austinmw]

Hi, I'm training with sagemaker using a custom docker image I created by extending the pytorch 1.6 training image with additional ssh settings for Horovod. I'm also using a custom estimator which I created by subclassing the pytorch estimator and adding a distribution parameter and configuration method.

When I launch single-node training everything works fine, but when I attempt to launch multi-node training, I get smdebug hook errors, even though I did not set any debugger rules. How can I turn off this functionality altogether?

[1,10]: File "/opt/conda/lib/python3.6/site-packages/torch/nn/modules/module.py", line 717, in _call_impl
[1,10]: hook = smd_utils.get_smdebug_hook()
[1,10]: File "/opt/conda/lib/python3.6/site-packages/torch/utils/smdebug.py", line 35, in get_smdebug_hook
[1,10]: return smd.get_hook(create_if_not_exists=True)
[1,10]: File "/opt/conda/lib/python3.6/site-packages/smdebug/pytorch/singleton_utils.py", line 22, in get_hook
[1,10]: create_if_not_exists=create_if_not_exists,
[1,10]: File "/opt/conda/lib/python3.6/site-packages/smdebug/core/singleton_utils.py", line 60, in get_hook
[1,10]: _create_hook(json_config_path, hook_class)
[1,10]: File "/opt/conda/lib/python3.6/site-packages/smdebug/core/singleton_utils.py", line 28, in _create_hook
[1,10]: hook = hook_class.create_from_json_file(json_file_path=json_config_path)
[1,10]: File "/opt/conda/lib/python3.6/site-packages/smdebug/core/hook.py", line 294, in create_from_json_file
[1,10]: return create_hook_from_json_config(cls, json_config_path=json_file_path)
[1,10]: File "/opt/conda/lib/python3.6/site-packages/smdebug/core/json_config.py", line 243, in create_hook_from_json_config
[1,10]: save_all=save_all,
[1,10]: File "/opt/conda/lib/python3.6/site-packages/smdebug/pytorch/hook.py", line 47, in init
[1,10]: include_workers=include_workers,
[1,10]: File "/opt/conda/lib/python3.6/site-packages/smdebug/core/hook.py", line 955, in init
[1,10]: include_workers=include_workers,
[1,10]: File "/opt/conda/lib/python3.6/site-packages/smdebug/core/hook.py", line 243, in init
[1,10]: self._initialize_to_last_saved_state()
[1,10]: File "/opt/conda/lib/python3.6/site-packages/smdebug/core/hook.py", line 251, in _initialize_to_last_saved_state
[1,10]: self.state_store = StateStore()
[1,10]: File "/opt/conda/lib/python3.6/site-packages/smdebug/core/state_store.py", line 42, in init
[1,10]: self._read_states_file()
[1,10]: File "/opt/conda/lib/python3.6/site-packages/smdebug/core/state_store.py", line 76, in _read_states_file
[1,10]: parameters = json.load(json_data)
[1,10]: File "/opt/conda/lib/python3.6/json/init.py", line 299, in load
[1,10]: parse_constant=parse_constant, object_pairs_hook=object_pairs_hook, **kw)
[1,10]: File "/opt/conda/lib/python3.6/json/init.py", line 354, in loads
[1,10]: return _default_decoder.decode(s)
[1,10]: File "/opt/conda/lib/python3.6/json/decoder.py", line 342, in decode
[1,10]: raise JSONDecodeError("Extra data", s, end)
[1,10]:json.decoder.JSONDecodeError: Extra data: line 1 column 1444 (char 1443)
[1,10]:Error in atexit._run_exitfuncs:
[1,10]:Traceback (most recent call last):
[1,10]: File "/opt/conda/lib/python3.6/site-packages/smdebug/core/hook.py", line 961, in _cleanup
[1,10]: if not self.exported_collections:
[1,10]:AttributeError: 'Hook' object has no attribute 'exported_collections'

[austinmw]

Uninstalling smdebug from the image worked. Upgrading smdebug to latest version did not work.

[Vikas-kum]

Hi @austinmw . Thanks for reporting this. Can you please provide the example script to reproduce this.

[austinmw]

I used the custom framework paradigm so it's a package of about 30 python files that I'm unable to share. Could try to put together a minimum reproducible example but it'd take a while.

The underlying model is partially based on https://github.com/ifzhang/FairMOT (using HRNet-18 backbone).

[amirhRahimi1993]

I have the same problem with pytorch in sagemaker. Although, my code works correctly in local device in sagemaker problem is emerged.

AttributeError Traceback (most recent call last)
in
15 # os.mkdir(PATH)
16 # shutil.copy(path,PATH)
---> 17 s = start_process(path,cp_path,isPath,isCuda)
18 K = s.pullThetrigger()
19 #report = open(os.path.join(PATH,"report.txt"),"w")

~/face_estimation_deployment/Invoke_AI.py in init(self, img_path, cp_path, iSpath, isCuda)
48 self.Cheek = "CHEEK"
49 self.requirement = ["CHINE","JAWLINE","SMOKELINE","KHATELABKHAND","AROUNDLIP","AROUNDEYE","FOREHEAD"]
---> 50 self.__check_value()
51 self.__resizer()
52 X = "numpy pandas gdown tqdm tensorflow keras mtcnn".split(" ")

~/face_estimation_deployment/Invoke_AI.py in __check_value(self)
63 subprocess.check_call([sys.executable, '-m', 'pip', 'install', package])
64 def __check_value(self):
---> 65 if self.checker.checker()==False:
66 print("Essentioal part of Face isnt Find the program wont execute")
67 return

~/face_estimation_deployment/segment_Face.py in checker(self)
1111 def checker(self):
1112 requirment , parsing , _ = self.__start_PROCESS()
-> 1113 ALLCLASSES = np.unique(parsing)
1114 for r in requirment:
1115 inner_correct = False

~/face_estimation_deployment/segment_Face.py in __start_PROCESS(self)
1107 img = img.cuda()
1108 out = net(img)[0]
-> 1109 parsing = out.squeeze(0).cpu().numpy().argmax(0)
1110 return requirment,parsing,image
1111 def checker(self):

/opt/conda/lib/python3.6/site-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
733 if is_hook_registration_suspended() is False:
734 import torch.utils.smdebug as smd_utils
--> 735 hook = smd_utils.get_smdebug_hook()
736 if hook and not hook.has_registered_module:
737 hook.register_hook(self)

/opt/conda/lib/python3.6/site-packages/torch/utils/smdebug.py in get_smdebug_hook()
47
48 try:
---> 49 import smdebug.pytorch as smd
50 except ImportError:
51 return None

/opt/conda/lib/python3.6/site-packages/smdebug/pytorch/init.py in
2 from smdebug import ReductionConfig, SaveConfig, SaveConfigMode, modes
3 from smdebug.core.collection import CollectionKeys
----> 4 from smdebug.trials import create_trial
5
6 # Local

/opt/conda/lib/python3.6/site-packages/smdebug/trials/init.py in
1 # Local
----> 2 from .local_trial import LocalTrial
3 from .s3_trial import S3Trial
4 from .trial import Trial
5 from .utils import create_trial

/opt/conda/lib/python3.6/site-packages/smdebug/trials/local_trial.py in
4 # First Party
5 from smdebug.core.collection_manager import CollectionManager
----> 6 from smdebug.core.index_reader import LocalIndexReader
7 from smdebug.core.utils import get_path_to_collections, list_collection_files_in_directory
8

/opt/conda/lib/python3.6/site-packages/smdebug/core/index_reader.py in
21 from smdebug.core.modes import ModeKeys
22 from smdebug.core.s3_utils import list_s3_objects
---> 23 from smdebug.core.tfrecord.tensor_reader import TensorReader
24 from smdebug.core.utils import (
25 get_path_to_events_directory,

/opt/conda/lib/python3.6/site-packages/smdebug/core/tfrecord/tensor_reader.py in
5 from smdebug.core.logger import get_logger
6 from smdebug.core.modes import MODE_PLUGIN_NAME, MODE_STEP_PLUGIN_NAME, ModeKeys
----> 7 from smdebug.core.tfevent.event_file_reader import get_tensor_data
8 from smdebug.core.tfevent.proto.event_pb2 import Event
9 from smdebug.core.tfrecord.record_reader import masked_crc32c

/opt/conda/lib/python3.6/site-packages/smdebug/core/tfevent/event_file_reader.py in
22
23 # First Party
---> 24 import smdebug.core.tfevent.proto.types_pb2 as types_pb2
25 from smdebug.core.modes import MODE_PLUGIN_NAME, MODE_STEP_PLUGIN_NAME, ModeKeys
26 from smdebug.core.tfrecord.record_reader import RecordReader

AttributeError: module 'smdebug' has no attribute 'core'