From 8a59e377d3631ae15bd6a2405c9cf7152895b2f5 Mon Sep 17 00:00:00 2001 From: Johannes Hjorth Date: Thu, 21 Sep 2023 12:45:45 +0200 Subject: [PATCH] Updated core.py, adding a wrapper function that takes args, while the underlying functions takes specific arguments (so they can be called from jupyter notebooks etc). --- snudda/cli.py | 20 +- snudda/core.py | 325 ++++++++++++-------- snudda/simulate/model_current_injections.py | 6 +- 3 files changed, 213 insertions(+), 138 deletions(-) diff --git a/snudda/cli.py b/snudda/cli.py index 48c5492c3..7b7cf536c 100644 --- a/snudda/cli.py +++ b/snudda/cli.py @@ -44,7 +44,7 @@ def snudda_cli(): help="Path to neurons_dir, default is $DATA/neurons (DEPRECATED, use --snudda_data instead") init_parser.add_argument("-overwrite", "--overwrite", help="Allow overwriting of old directory", action="store_true") - init_parser.add_argument("-connectionFile", "--connectionFile", default=None, + init_parser.add_argument("-connectionFile", "--connectionFile", default=None, dest="connection_file", help="Use connectivity from user specified JSON file") init_parser.add_argument("-randomseed", "--randomseed", "--seed", default=None, help="Random seed", type=int) init_parser.add_argument("--profile", help="Run python cProfile", action="store_true") @@ -74,7 +74,6 @@ def snudda_cli(): detect_parser.add_argument("-parallel", "--parallel", action="store_true", default=False) detect_parser.add_argument("-ipython_profile", "--ipython_profile", default=None) - prune_parser = sub_parsers.add_parser("prune") prune_parser.add_argument("path", help="Location of network") prune_parser.add_argument("-randomseed", "--randomseed", "--seed", default=None, help="Random seed", type=int) @@ -90,7 +89,6 @@ def snudda_cli(): prune_parser.add_argument("-parallel", "--parallel", action="store_true", default=False) prune_parser.add_argument("-ipython_profile", "--ipython_profile", default=None) - input_parser = sub_parsers.add_parser("input") input_parser.add_argument("path", help="Location of network") input_parser.add_argument("--input", help="Input json config file (for input setup)", default=None) @@ -154,15 +152,15 @@ def snudda_cli(): snudda = Snudda(args.path) - actions = {"init": snudda.init_config, - "place": snudda.place_neurons, - "detect": snudda.touch_detection, - "prune": snudda.prune_synapses, - "input": snudda.setup_input, - "export": snudda.export_to_SONATA, - "convert": snudda.export_to_SONATA, + actions = {"init": snudda.init_config_wrapper, + "place": snudda.place_neurons_wrapper, + "detect": snudda.touch_detection_wrapper, + "prune": snudda.prune_synapses_wrapper, + "input": snudda.setup_input_wrapper, + "export": snudda.export_to_SONATA_wrapper, + "convert": snudda.export_to_SONATA_wrapper, "analyse": snudda.analyse, - "simulate": snudda.simulate, + "simulate": snudda.simulate_wrapper, "help": snudda.help_info} if not hasattr(args, 'ipython_profile'): diff --git a/snudda/core.py b/snudda/core.py index c0538edca..d7c748ef4 100755 --- a/snudda/core.py +++ b/snudda/core.py @@ -90,7 +90,8 @@ def help_info(args): ############################################################################ - def init_config(self, args): + def init_config_wrapper(self, args): + """ Creates network-config.json in network_path. @@ -100,14 +101,29 @@ def init_config(self, args): Example: snudda init -size 100 [-overwrite] [-randomseed 1234] [--profile] [--verbose] path """ + + assert args.size is not None, "You need to specify --size when initialising config for the network" + + self.init_config(network_size=args.size, + snudda_data=args.snudda_data, + neurons_dir=args.neurons_dir, + connection_file=args.connection_file, + overwrite=args.overwrite, + random_seed=args.randomseed) + + def init_config(self, network_size, + snudda_data=None, + neurons_dir=None, + connection_file=None, + overwrite=False, + random_seed=None): + # self.networkPath = args.path print("Creating config file") print(f"Network path: {self.network_path}") - assert args.size is not None, "You need to specify --size when initialising config for the network" - from snudda.init.init import SnuddaInit - struct_def = {"Striatum": args.size, + struct_def = {"Striatum": network_size, "GPe": 0, "GPi": 0, "SNr": 0, @@ -116,30 +132,28 @@ def init_config(self, args): "Thalamus": 0} # Cortex and thalamus axons disabled right now, set to 1 to include one - if not args.overwrite: + if not overwrite: assert not os.path.exists(self.network_path), \ (f"Network path {self.network_path} already exists (aborting to prevent accidental overwriting)." "\nCall snudda init with --overwrite to override and overwrite the old data.") self.make_dir_if_needed(self.network_path) - random_seed = args.randomseed - config_file = os.path.join(self.network_path, "network-config.json") SnuddaInit(struct_def=struct_def, - neurons_dir=args.neurons_dir, - snudda_data=args.snudda_data, + neurons_dir=neurons_dir, + snudda_data=snudda_data, config_file=config_file, random_seed=random_seed, - connection_override_file=args.connectionFile) + connection_override_file=connection_file) - if args.size > 1e5: + if network_size > 1e5: print(f"Make sure there is enough disk space in {self.network_path}") print("Large networks take up ALOT of space") ############################################################################ - def place_neurons(self, args): + def place_neurons_wrapper(self, args): """ Places neurons in 3D space. Creates network-neuron-positions.hdf5 in network_path. @@ -150,32 +164,46 @@ def place_neurons(self, args): snudda place [--raytraceBorders] [--profile] [--verbose] [--h5legacy] [-parallel] path """ + + if args.h5legacy: + h5libver = "earliest" + else: + h5libver = "latest" # default + + self.place_neurons(random_seed=args.randomseed, + parallel=args.parallel, + ipython_profile=args.ipython_profile, + raytrace_borders=args.raytrace_borders, + h5libver=h5libver, + verbose=args.verbose) + + def place_neurons(self, + random_seed=None, + parallel=False, + ipython_profile=None, + raytrace_borders=False, + h5libver="latest", + verbose=False): + # self.networkPath = args.path print("Placing neurons") print(f"Network path: {self.network_path}") log_file_name = os.path.join(self.network_path, "log", "place-neurons.txt") - random_seed = args.randomseed - self.setup_log_file(log_file_name) # sets self.logFile - if args.parallel: - self.setup_parallel(ipython_profile=args.ipython_profile) # sets self.d_view + if parallel: + self.setup_parallel(ipython_profile=ipython_profile) # sets self.d_view from snudda.place.place import SnuddaPlace - if args.h5legacy: - h5libver = "earliest" - else: - h5libver = "latest" # default - sp = SnuddaPlace(network_path=self.network_path, log_file=self.logfile, - verbose=args.verbose, + verbose=verbose, d_view=self.d_view, h5libver=h5libver, - raytrace_borders=args.raytrace_borders, + raytrace_borders=raytrace_borders, random_seed=random_seed) sp.place() @@ -187,7 +215,7 @@ def place_neurons(self, args): ############################################################################ - def touch_detection(self, args): + def touch_detection_wrapper(self, args): """ Synapse touch detection. Writes results to network_path/voxels (one file per hypervoxel). Also adds synapse projections between structures. @@ -200,19 +228,39 @@ def touch_detection(self, args): snudda detect [-cont] [-hvsize HVSIZE] [--volumeID VOLUMEID] [--profile] [--verbose] [--h5legacy] [-parallel] path """ - # self.networkPath = args.path - print("Touch detection") - print("Network path: " + str(self.network_path)) if args.hvsize is not None: hyper_voxel_size = int(args.hvsize) else: hyper_voxel_size = 100 - if args.volumeID is not None: - volume_id = args.volumeID + if args.h5legacy: + h5libver = "earliest" else: - volume_id = None + h5libver = "latest" # default + + self.touch_detection(random_seed=args.randomseed, + parallel=args.parallel, + ipython_profile=args.ipython_profile, + hyper_voxel_size=hyper_voxel_size, + volume_id=args.volumeID, + h5libver=h5libver, + verbose=args.verbose, + cont=args.cont) + + def touch_detection(self, + random_seed=None, + parallel=False, + ipython_profile=None, + hyper_voxel_size=100, + volume_id=None, + h5libver="latest", + verbose=False, + cont=False): + + # self.networkPath = args.path + print("Touch detection") + print("Network path: " + str(self.network_path)) log_dir = os.path.join(self.network_path, "log") if not os.path.exists(log_dir): @@ -224,20 +272,13 @@ def touch_detection(self, args): log_filename = os.path.join(self.network_path, "log", "touch-detection.txt") save_file = os.path.join(self.network_path, "voxels", "network-putative-synapses.hdf5") - random_seed = args.randomseed - voxel_dir = os.path.join(self.network_path, "voxels") self.make_dir_if_needed(voxel_dir) self.setup_log_file(log_filename) # sets self.logfile - if args.parallel: - self.setup_parallel(ipython_profile=args.ipython_profile) # sets self.d_view - - if args.h5legacy: - h5libver = "earliest" - else: - h5libver = "latest" # default + if parallel: + self.setup_parallel(ipython_profile=ipython_profile) # sets self.d_view from snudda.detect.detect import SnuddaDetect @@ -253,9 +294,9 @@ def touch_detection(self, args): hyper_voxel_size=hyper_voxel_size, h5libver=h5libver, random_seed=random_seed, - verbose=args.verbose) + verbose=verbose) - if args.cont: + if cont: # Continue previous run print("Continuing previous touch detection") sd.detect(restart_detection_flag=False) @@ -276,7 +317,8 @@ def touch_detection(self, args): ############################################################################ - def prune_synapses(self, args): + def prune_synapses_wrapper(self, args): + """ Merges data from synapse detection, then does synapse pruning. Writes results to network-synapses.hdf5 in network_path. @@ -287,43 +329,57 @@ def prune_synapses(self, args): snudda prune [--configFile CONFIG_FILE] [--profile] [--verbose] [--h5legacy] [--keepfiles] [-parallel] path """ + if args.h5legacy: + h5libver = "earliest" + else: + h5libver = "latest" # default + + self.prune_synapses(config_file=args.config_file, + random_seed=args.randomseed, + parallel=args.parallel, + ipython_profile=args.ipython_profile, + verbose=args.verbose, + keep_files=args.keepfiles, + save_putative_synapses = args.savePutative) + + def prune_synapses(self, + config_file, + random_seed=None, parallel=False, ipython_profile=None, + h5libver="latest", + verbose=False, + keep_files=False, + save_putative_synapses=False): + # self.networkPath = args.path print("Prune synapses") - print("Network path: " + str(self.network_path)) + print(f"Network path: {self.network_path}") from snudda.detect.prune import SnuddaPrune log_filename = os.path.join(self.network_path, "log", "synapse-pruning.txt") - random_seed = args.randomseed - self.setup_log_file(log_filename) # sets self.logfile - if args.parallel: - self.setup_parallel(ipython_profile=args.ipython_profile) # sets self.d_view + if parallel: + self.setup_parallel(ipython_profile=ipython_profile) # sets self.d_view # Optionally set this scratch_path = None - if args.h5legacy: - h5libver = "earliest" - else: - h5libver = "latest" # default - sp = SnuddaPrune(network_path=self.network_path, logfile=self.logfile, logfile_name=log_filename, - config_file=args.config_file, + config_file=config_file, d_view=self.d_view, scratch_path=scratch_path, h5libver=h5libver, random_seed=random_seed, - verbose=args.verbose, - keep_files=args.keepfiles or args.savePutative) + verbose=verbose, + keep_files=keep_files or save_putative_synapses) sp.prune() - if args.savePutative: + if save_putative_synapses: sp.save_putative_synapses() self.cleanup_workers() @@ -333,7 +389,7 @@ def prune_synapses(self, args): ############################################################################ - def setup_input(self, args): + def setup_input_wrapper(self, args): """ Creates synaptic input for network based on input.json, writes input-spikes.hdf5 in network_path. Args: @@ -343,18 +399,44 @@ def setup_input(self, args): snudda input [--input INPUT] [--inputFile INPUT_FILE] [--networkFile NETWORK_FILE] [--time TIME] [-randomseed 123] [--profile] [--verbose] [--h5legacy] [-parallel] path """ + if args.h5legacy: + h5libver = "earliest" + else: + h5libver = "latest" # default + + self.setup_input(network_file=args.network_file, + input_file=args.input_file, + input_config=args.input, + input_time=args.time, + random_seed=args.randomseed, + use_meta_input=not args.no_meta_input, + h5libver=h5libver, + parallel=args.parallel, + ipython_profile=args.ipython_profile, + verbose=args.verbose) + + def setup_input(self, + network_file=None, + input_file=None, + input_config=None, + input_time=None, + use_meta_input=True, + random_seed=None, + h5libver="latest", + parallel=False, + ipython_profile=None, + verbose=False): + print("Setting up inputs, assuming input.json exists") log_filename = os.path.join(self.network_path, "log", "setup-input.txt") self.setup_log_file(log_filename) # sets self.logfile - if args.parallel: - self.setup_parallel(ipython_profile=args.ipython_profile) # sets self.d_view + if parallel: + self.setup_parallel(ipython_profile=ipython_profile) # sets self.d_view from snudda.input.input import SnuddaInput - if "input" in args and args.input: - input_config = args.input - else: + if input_config is None: input_config = os.path.join(self.network_path, "input.json") snudda_data = get_snudda_data(network_path=self.network_path) @@ -362,41 +444,23 @@ def setup_input(self, args): print(f"Missing input config file: {input_config}") return - if args.network_file: - network_file = args.network_file - else: + if network_file is None: network_file = os.path.join(self.network_path, "network-synapses.hdf5") - if args.input_file: - spike_file = args.input_file - else: - spike_file = os.path.join(self.network_path, "input-spikes.hdf5") - - if args.time: - input_time = args.time - else: - input_time = None - - random_seed = args.randomseed - - if args.h5legacy: - h5libver = "earliest" - else: - h5libver = "latest" # default + if input_file is None: + input_file = os.path.join(self.network_path, "input-spikes.hdf5") - print(f"Writing input spikes to {spike_file}") - - use_meta_input = not args.no_meta_input + print(f"Writing input spikes to {input_file}") si = SnuddaInput(input_config_file=input_config, hdf5_network_file=network_file, - spike_data_filename=spike_file, + spike_data_filename=input_file, time=input_time, logfile=self.logfile, rc=self.rc, random_seed=random_seed, h5libver=h5libver, - verbose=args.verbose, + verbose=verbose, use_meta_input=use_meta_input) si.generate() @@ -407,7 +471,7 @@ def setup_input(self, args): ############################################################################ - def export_to_SONATA(self, args): + def export_to_SONATA_wrapper(self, args): """ Export network to SONATA files. (Currently not functioning) @@ -415,19 +479,20 @@ def export_to_SONATA(self, args): args : command line arguments from argparse """ + self.export_to_SONATA(network_file=args.network_file, + input_file=args.input_file) + + def export_to_SONATA(self, network_file=None, input_file=None): + from snudda.utils.export_sonata import ExportSonata print("Exporting to SONATA format") print(f"Network path: {self.network_path}") - if args.network_file: - network_file = args.network_file - else: + if network_file is None: network_file = os.path.join(self.network_path, "network-synapses.hdf5") - if args.input_file: - input_file = args.input_file - else: + if input_file is None: input_file = os.path.join(self.network_path, "input-spikes.hdf5") out_dir = os.path.join(self.network_path, "SONATA") @@ -484,7 +549,7 @@ def compile_mechanisms(mech_dir=None, snudda_data=None): ############################################################################ - def simulate(self, args): + def simulate_wrapper(self, args): """ Simulate network. Writes results to network_path/simulation. @@ -497,25 +562,43 @@ def simulate(self, args): [-mechdir MECH_DIR] [--profile] [--verbose] [--exportCoreNeuron] path """ + print(f"args: {args}") + + self.simulate(network_file=args.network_file, input_file=args.input_file, + output_file=args.output_file, snudda_data=args.snudda_data, + time=args.time, + mech_dir=args.mech_dir, neuromodulation=args.neuromodulation, + disable_synapses=args.disable_synapses, + disable_gj=args.disable_gj, + record_volt=args.record_volt, + record_all=args.record_all, + export_core_neuron=args.exportCoreNeuron, + verbose=args.verbose) + + def simulate(self, network_file=None, input_file=None, output_file=None, + snudda_data=None, + time=None, + mech_dir=None, neuromodulation=None, + disable_synapses=False, + disable_gj=False, + record_volt=False, + record_all=False, + export_core_neuron=False, + verbose=False): + start = timeit.default_timer() from mpi4py import MPI # This must be imported before neuron, to run parallel from neuron import h pc = h.ParallelContext() - if args.network_file: - network_file = args.network_file - else: + if network_file is None: network_file = os.path.join(self.network_path, "network-synapses.hdf5") - if args.input_file: - input_file = args.input_file - else: + if input_file is None: input_file = os.path.join(self.network_path, "input-spikes.hdf5") - if args.output_file: - output_file = args.output_file - else: + if output_file is None: output_file = os.path.join(self.network_path, "simulation", "output.hdf5") self.make_dir_if_needed(os.path.join(self.network_path, "simulation")) @@ -528,20 +611,18 @@ def simulate(self, args): # Problems with nested symbolic links when the second one is a relative # path going beyond the original base path - if args.mech_dir: - mech_dir = args.mech_dir - else: + if mech_dir is None: # Take into account which SNUDDA_DATA the user wants to use from snudda.utils.snudda_path import get_snudda_data - snudda_data = get_snudda_data(snudda_data=args.snudda_data, + snudda_data = get_snudda_data(snudda_data=snudda_data, network_path=self.network_path) mech_dir = os.path.realpath(snudda_path.snudda_parse_path(os.path.join("$DATA", "neurons", "mechanisms"), snudda_data)) - if args.neuromodulation is not None: + if neuromodulation is not None: # read neuromod file and determine if it is replay or adaptive, then if and import the correct one - with open(args.neuromodulation, "r") as f: + with open(neuromodulation, "r") as f: neuromod_dict = json.load(f, object_pairs_hook=OrderedDict) if "adaptive" in neuromod_dict["type"]: @@ -562,13 +643,9 @@ def simulate(self, args): if slurm_id is None: slurm_id = str(666) - print(f"args: {args}") - - disable_gj = args.disable_gj if disable_gj: print("!!! WE HAVE DISABLED GAP JUNCTIONS !!!") - disable_synapses = args.disable_synapses if disable_synapses: print("!!! SYNAPSES DISABLED") @@ -579,15 +656,15 @@ def simulate(self, args): print(f"Creating directory {log_dir}") os.makedirs(log_dir, exist_ok=True) - if args.neuromodulation is not None: + if neuromodulation is not None: # read neuromod file and determine if it is replay or adaptive, then if and import the correct one - with open(args.neuromodulation, 'r') as neuromod_f: + with open(neuromodulation, 'r') as neuromod_f: neuromod_dict = json.load(neuromod_f, object_pairs_hook=OrderedDict) if 'type' not in neuromod_dict: - print(f"Neuromodulation is not defined correctly in {args.neuromodulation} : 'type' is missing. " + print(f"Neuromodulation is not defined correctly in {neuromodulation} : 'type' is missing. " f"Did you specify the correct file?") sys.exit(-1) @@ -600,7 +677,7 @@ def simulate(self, args): disable_gap_junctions=disable_gj, disable_synapses=disable_synapses, log_file=log_file, - verbose=args.verbose) + verbose=verbose) sim.setup() sim.add_external_input() @@ -617,7 +694,7 @@ def simulate(self, args): disable_synapses=disable_synapses, log_file=log_file, neuromodulator_description=neuromod_dict, - verbose=args.verbose) + verbose=verbose) sim.setup() sim.add_external_input() @@ -633,25 +710,25 @@ def simulate(self, args): disable_gap_junctions=disable_gj, disable_synapses=disable_synapses, log_file=log_file, - verbose=args.verbose) + verbose=verbose) sim.setup() sim.add_external_input() sim.check_memory_status() - if args.record_volt: + if record_volt: # sim.add_volt_recording_all() sim.add_volt_recording_soma() # sim.addRecordingOfType("dSPN",5) # Side len let you record from a subset - if args.record_all: - record_cell_id = np.array([int(x) for x in args.record_all.split(",")]) + if record_all: + record_cell_id = np.array([int(x) for x in record_all.split(",")]) sim.add_volt_recording_all(cell_id=record_cell_id) sim.add_synapse_current_recording_all(record_cell_id) - t_sim = args.time * 1000 # Convert from s to ms for Neuron simulator + t_sim = time * 1000 # Convert from s to ms for Neuron simulator - if args.exportCoreNeuron: + if export_core_neuron: sim.export_to_core_neuron() return # We do not run simulation when exporting to core neuron diff --git a/snudda/simulate/model_current_injections.py b/snudda/simulate/model_current_injections.py index 3bada4ea2..acaef438f 100644 --- a/snudda/simulate/model_current_injections.py +++ b/snudda/simulate/model_current_injections.py @@ -333,9 +333,9 @@ def __init__(self): args = FakeArgs() - sn.place_neurons(args) - sn.touch_detection(args) - sn.prune_synapses(args) + sn.place_neurons_wrapper(args) + sn.touch_detection_wrapper(args) + sn.prune_synapses_wrapper(args) ############################################################################