Clean up differences between neuron and neuron-nightly (#77)

* Cleanup neuron vs. neuron-nightly

Note that the actual diffs are mainly due to differences in error
formatting and reporting. The hope is that it might be easier to fix the
errors than to otherwise hide the errors as false positive differences.

Many of the errors are simply because the models' mechanism files are
not being found and compiled, so the HOC scripts are launched with nrniv
and there are errors due to mechanisms not being defined. Setting the
path to the .mod files was needed for models: 83344, 83590, 84589,
94321, 97860, 105385, 108458, 108459, 114359, 117207, 118631, 140471,
140828, 143604, 144482, 146026, 150239, 151443, 168310, 184054, 187473,
195666, 237594, 254217, 256024, 260178 and 260967.

Some fixes were needed to the models themselves. These are contained in
the pull requests ModelDBRepository/83344#1, ModelDBRepository/118631#1,
ModelDBRepository/146949#3, ModelDBRepository/150245#3 and
ModelDBRepository/267067#3.

Finally, the other fixes needed to eliminate the diff between NEURON
8.2.2 and neuron-nightly (the current development branch) were:
* 33975: suppress error message that wasn't printed in older versions
* 149739: fix file encoding, create output directories, reduce tstop, do
          not try to call nonexistent demo_run() function
* 168314: fix paths so it runs on case-sensitive filesystems
* 184054: reduce tstop
* 187615: do not run, tries to open nonexistent bac6.ses file
* 189347: do not try to dynamically load mechanisms; we launch via
          special so they are already loaded
* 222359: rename file so it runs on case-sensitive filesystems
* 229276: create output directory, comment out problematic line that did
          not seem to be executed
* 232097: create output directories
* 249463: do not run, tries to open nonexistent bac6.ses file
* 258867: do not run, tries to open C:/.. path not included in model
* 266818: add missing includes of stdlib.hoc and stdgui.hoc
* 267293: do not run, tries to open nonexistent bac6.ses file
* 267384: declare transvec variable with hopefully-large-enough size
* 267594: do not run, seems to depend on files not included in the model

* Report meaningful pipeline status, check gout (#80)

Compress JSON files that include gout information.

* runmodels: include model ID in working directory

Not all models in the database have unique directory names embedded in
their .zip files.

* ci: drop --virtual as Xvfb is launched manually

.github/workflows/nrn-modeldb-ci.yaml

@@ -162,7 +162,14 @@ jobs:
           python -m pip install $NEURON_V1
         fi
         nrn_ver=`python -c "from neuron import __version__ as nrn_ver; print(nrn_ver)"`
-        runmodels --virtual --workdir=$nrn_ver $MODELS_TO_RUN
+        ps uxf # debug
+        runmodels --gout --workdir=$nrn_ver $MODELS_TO_RUN
+        # Filter out the gout data before generating HTML reports. The HTML
+        # diff uses the original gout files on disk anyway. Compress the large
+        # JSON file including gout data for inclusion in the artifacts
+        mv ${nrn_ver}.json ${nrn_ver}-full.json
+        jq -r 'del(.[].gout)' ${nrn_ver}-full.json > ${nrn_ver}.json
+        xz ${nrn_ver}-full.json
         report2html ${nrn_ver}.json
         if [[ -d "${DROP_DIR_V1}" ]]; then
           python -m pip uninstall --yes neuron-nightly==${nrn_ver}
@@ -182,7 +189,14 @@ jobs:
           python -m pip install $NEURON_V2
         fi
         nrn_ver=`python -c "from neuron import __version__ as nrn_ver; print(nrn_ver)"`
-        runmodels --virtual --workdir=$nrn_ver $MODELS_TO_RUN
+        ps uxf # debug
+        runmodels --gout --workdir=$nrn_ver $MODELS_TO_RUN
+        # Filter out the gout data before generating HTML reports. The HTML
+        # diff uses the original gout files on disk anyway. Compress the large
+        # JSON file including gout data for inclusion in the artifacts
+        mv ${nrn_ver}.json ${nrn_ver}-full.json
+        jq -r 'del(.[].gout)' ${nrn_ver}-full.json > ${nrn_ver}.json
+        xz ${nrn_ver}-full.json
         report2html ${nrn_ver}.json
         if [[ -d "${DROP_DIR_V2}" ]]; then
           python -m pip uninstall --yes neuron-nightly==${nrn_ver}
@@ -197,8 +211,9 @@ jobs:
         diffreports2html ${nrn1_ver}.json ${nrn2_ver}.json
 
     - uses: actions/upload-artifact@v3
+      if: ${{ always() }}
       with:
         name: ${{ env.nrn1_ver }}-vs-${{ env.nrn2_ver }}
         path: |
-          ./*.json
+          ./*.json.xz
           ./*.html

modeldb/commands.py

@@ -222,7 +222,7 @@ def diffreports2html(args=None):
     report_filename = os.path.join(Path(json_report1).resolve().parent, report_title + '.html')
     runtime_report_title = 'Runtimes ' + report_title
     runtime_report_filename = os.path.join(Path(json_report1).resolve().parent, "runtimes-" + report_title + '.html')
-    diff_dict, gout_dict,runtime_dict, v1, v2 = diff_reports(json_report1, json_report2)
+    diff_dict, gout_dict, runtime_dict, stats_dict, v1, v2 = diff_reports(json_report1, json_report2)
 
     print('Writing {} ...'.format(report_filename))
     with open(report_filename, 'w') as fh:
@@ -242,3 +242,53 @@ def diffreports2html(args=None):
             v2=v2),
     )
     print('Done.')
+    # Return a useful status code
+    code = 0
+    if len(diff_dict) > 1:
+        assert "0" in diff_dict  # summary info; not a real diff
+        print("FAILURE: stdout diffs in {}".format(set(diff_dict.keys()) - {"0"}))
+        code = 1
+    if len(gout_dict) > 1:
+        assert "0" in gout_dict  # summary info; not a real diff
+        print("FAILURE: gout diffs in {}".format(set(diff_dict.keys()) - {"0"}))
+        code = 1
+    total_failures = sum(
+        version_stats["Failed models"]["Count"] for version_stats in stats_dict.values()
+    )
+    if total_failures > 0:
+        print(
+            "FAILURE: there were {} failed model builds across {} versions of NEURON".format(
+                total_failures, len(stats_dict)
+            )
+        )
+        code = 1
+    total_runtime_failures = sum(
+        version_stats["Failed runs"]["Count"] for version_stats in stats_dict.values()
+    )
+    if total_runtime_failures > 0:
+        print(
+            "FAILURE: there were {} failed model runs across {} versions of NEURON".format(
+                total_runtime_failures, len(stats_dict)
+            )
+        )
+        code = 1
+    # These are not expected to be different between the two NEURON versions tested
+    assert (
+        len(
+            {
+                version_stats["Skipped runs"]["Count"]
+                for version_stats in stats_dict.values()
+            }
+        )
+        == 1
+    )
+    assert (
+        len(
+            {
+                version_stats["Total nof models run"]
+                for version_stats in stats_dict.values()
+            }
+        )
+        == 1
+    )
+    return code

modeldb/modeldb-run.yaml

@@ -340,6 +340,9 @@
     - cat mosinit.hoc >> temp
     - mv temp  mosinit.hoc
 33975:
+    curate_patterns:
+    - pattern: 'hoc_run1: caught exception: hoc_execerror: We were unable to associate a PlayRecord item with a RANGE variable'
+      repl: ''
     run:
     - ringperf()
     - run()
@@ -504,10 +507,12 @@
     - run()
     - verify_graph_()
 83590:
+    model_dir: mechanisms
     run: []
     script:
     - sed -i'.bak' -e 's/mustart  = -0.400/mustart  = 0.800/g;s/sigmastart  = 0.064/sigmastart  = 0.128/g' mylibs/TFparams.hoc
 84589:
+    model_dir: mod
     run:
     - //ignore sim in subdirectory
     - // have not been able to run yet
@@ -561,6 +566,8 @@
 260971:
     comment: tricky to set-up due to mosinit.hoc autodiscovery
     skip: true
+83344:
+    model_dir: mod
 84655:
     run:
     - run()
@@ -605,6 +612,7 @@
     - restart("shortRun")
     - verify_graph_()
 94321:
+    model_dir: mechanisms
     run:
     - tstop=10
     - init()
@@ -635,6 +643,7 @@
     # avoid printing out times
     - sed -i'.bak' -e 's/^startsw()$/{startsw()}/g' ga_setup.hoc
 97860:
+    model_dir: mod
     run:
     - run()
     - verify_graph_()
@@ -676,6 +685,7 @@
     - fig9A1()
     - verify_graph_()
 105385:
+    model_dir: mod
     run:
     - loadfig9()
     - tstop=100
@@ -703,12 +713,14 @@
     - runeg()
     - verify_graph_()
 108458:
+    model_dir: mod
     run:
     - AP3_toq = 1
     - set_stim()
     - run()
     - verify_graph_()
 108459:
+    model_dir: mod
     run:
     - run()
     - verify_graph_()
@@ -781,6 +793,8 @@
     - tstop_changed()
     - run()
     - verify_graph_()
+114359:
+    model_dir: mod
 114394:
     run:
     - run()
@@ -865,6 +879,7 @@
     - run()
     - verify_graph_()
 117207:
+    model_dir: Model
     run:
     - restart("AckerAnticBasalBackprop")
     - RunAll()
@@ -886,6 +901,7 @@
     - Graph[0].exec_menu("View = plot")
     - verify_graph_()
 118631:
+    model_dir: mechanisms
     run:
     - use_mcell_ran4(1)
     - graph_and_run()
@@ -1004,6 +1020,10 @@
 139656:
     comment: //do not run
     run: null
+140471:
+    model_dir: mods
+140828:
+    model_dir: 'mod.files'
 140881:
     comment: //do not run Lytton model takes too long to run and complex to edit
     run: null
@@ -1018,8 +1038,12 @@
 141505:
     comment: //do not run Specialized installation environment
     run: null
+143604:
+    model_dir: mod
 144385:
-  model_dir: mod
+    model_dir: mod
+144482:
+    model_dir: mechanism/mechanism_cell1
 144523:
     run:
     - objref test_graph_
@@ -1043,6 +1067,8 @@
     run:
     - ParmFitnessGui[0].run()
     - verify_graph_()
+146026:
+    model_dir: channels
 146030:
     curate_patterns:
       - pattern: 'Setup time for simulation: [0-9]+\.[0-9]+  seconds'
@@ -1069,8 +1095,14 @@
     run: null
 149739:
     run:
-    - demo_run()
     - verify_graph_()
+    script:
+    - mkdir Connection SP
+    - mv OB.hoc OB.hoc.iconv.bak
+    - iconv -f LATIN1 -t UTF-8 OB.hoc.iconv.bak > OB.hoc
+    - sed -i'.bak' 's#tstop   = 3000#tstop   = 10#g' OB.hoc
+150239:
+    model_dir: nrn/mod
 150245:
     curate_patterns:
       - pattern: 't=100\.00;122\([0-9]+\.[0-9]+\)'
@@ -1083,48 +1115,96 @@
 150284:
     comment: //do not run Mattione Le Novere missing _run_me.hoc in parallel model
     run: null
+151443:
+    model_dir: channels
 167772:
     comment: //do not run Tried "initDialog.unmap(1)" - didnt work.  Need to change
         archive
     run: null
+168310:
+    model_dir: experiment
+168314:
+    script:
+    # fixup for case-sensitive filesystems
+    - sed -i'.bak' 's#path = "./inputs/"#path = "./Inputs/"#g' ReadExperiments.hoc
+    - sed -i'.bak' 's#load_file("protocol.hoc")#load_file("Protocol.hoc")#g' Hipp.hoc
+184054:
+    model_dir: fullMorphCaLTP8
+    script:
+    - sed -i'.bak' 's#tstop = 250#tstop = 25#g' fullMorphCaLTP8/doTBSStimCC.hoc
+187473:
+    model_dir: mods
 187604:
   curate_patterns:
     - pattern: 'TIME HOST 0: [0-9\.]+ seconds \(set up\)'
       repl: 'TIME HOST 0: %elapsed_time% seconds (set up)'
     - pattern: 'TIME HOST 0: [0-9\.]+ seconds \(created cells\)'
       repl: 'TIME HOST 0: %elapsed_time% seconds (created cells)'
+187615:
+    comment: Tries to open bac6.ses, which does not exist
+    run: null
+189347:
+    script:
+    # we launch with special so don't try and dlopen mechanisms too
+    - sed -i'.bak' 's#nrn_load_dll#// nrn_load_dll#g' main.hoc
+195666:
+    model_dir: mod_files
 206244:
     run:
     - chdir("mechanism")
     - xopen("mosinit.hoc")
     - unix_spike_atten_hof()
     - verify_graph_()
+222359:
+    script:
+    # Fix for case-sensitive filesystems
+    - mv Voltage.ses voltage.ses
 223962:
     skip: true
     comment: takes too long, need to see how to reduce time
 229276:
     model_dir: mechanisms
     script:
+    - mkdir hoc_recordings
     - cp template.hoc template.hoc.iconv.bak
     - cp createsimulation.hoc createsimulation.hoc.iconv.bak
     - iconv -f LATIN1 -t UTF-8 template.hoc.iconv.bak > template.hoc
     - iconv -f LATIN1 -t UTF-8 createsimulation.hoc.iconv.bak > createsimulation.hoc
     - sed -i'.bak' -e 's#^tstop=\([0-9a-z+]*\)#tstop=((\1)/500)#g' createsimulation.hoc
+    - sed -i'.bak' 's#\(cell.synapses.update_synapses(synapse_plot)\)#// \1#g' init_super.hoc
+232097:
+    script:
+    - mkdir connection input
+237594:
+    model_dir: mechanism
 244262:
     script:
     - if [[ ! -f Iintra.dat ]]; then unzip Iintra.dat.zip; fi
     - ls -l Iintra.dat
+249463:
+    comment: Tries to open bac6.ses, which does not exist
+    run: null
+254217:
+    model_dir: _mod
+256024:
+    model_dir: mods
 149174:
     skip: true
     comment: testing in Python + hardcoded NEURON
 258867:
-    run:
-    - run()
-    - verify_graph_()
+    run: null
     script:
     - sed -i'.bak' -e 's/tstop = /tstop = 50\/\//' Fig8_tuft_NMDA_spike.hoc
     - sed -i'.bak' -e 's/simul_iter=/simul_iter=2 \/\//' Fig8_tuft_NMDA_spike.hoc
-    comment: usual simulation time is ~ 10 minutes (tstop = 120 and simul_iter=10)
+    comment: 'Usual simulation time is ~ 10 minutes (tstop = 120 and simul_iter=10), skipped because it tries to load C:/Daten/Modeling/Alpha5_NMDA_single_comp/shared/ATF.hoc'
+260178:
+    model_dir: _mod
+260967:
+    model_dir: lib
+266818:
+    script:
+    - sed -i'.bak' '1s/^/{load_file("stdlib.hoc")}\n/' hoc_tools/nav_states.hoc
+    - sed -i'.bak' '1s/^/{load_file("stdgui.hoc")}\n/' hoc_tools/plots.hoc
 267067:
     # This model ID seems to have two ~different models inside it. Arbitrarily choose one.
     model_dir: 'Na12 Analysis/mechanisms'
@@ -1135,8 +1215,18 @@
     - (cd "Na12 Analysis" && mv mosinit.hoc mosinit.hoc.bak && ln -s runModel.hoc mosinit.hoc)
     - sed -i'.bak' -e 's#\./mosinit\.hoc#./mosinit.hoc.bak#g' 'Na12 Analysis/runModel.hoc'
     - sed -i'.bak' -e 's#^tstop=30000#tstop=100#g' 'Na12 Analysis/constants.hoc'
+267293:
+    comment: Tries to open bac6.ses, which does not exist
+    run: null
 267384:
     model_dir: 'BBP_TTPC_EXAMPLE'
+    script:
+    # 12 is the largest value (runModel-template.hoc); this is a guess
+    - sed -i'.bak' 's#load_file("nrngui.hoc")#objref transvec\ntransvec = new Vector(12)\nload_file("nrngui.hoc")#' BBP_TTPC_EXAMPLE/mosinit.hoc
+267594:
+    comment: 'Seems to be missing files: parinit.hoc, .mod file for h_ca, ...?'
+    model_dir: mods
+    run: null
 206267:
     run:
     - run()

modeldb/modelrun.py

@@ -209,16 +209,18 @@ def prepare_model(model):
             os.path.join(MODELS_ZIP_DIR, str(model.id) + ".zip"), "r"
     ) as zip_ref:
         model_dir = os.path.join(
-            model.working_dir, os.path.dirname(zip_ref.infolist()[0].filename)
+            model.working_dir,
+            str(model.id),
+            os.path.dirname(zip_ref.infolist()[0].filename),
         )
-        model_run_info_file = os.path.join(model_dir, str(model.id) + '.yaml')
+        model_run_info_file = os.path.join(model_dir, str(model.id) + ".yaml")
         if model._inplace and os.path.isfile(model_run_info_file):
             with open(model_run_info_file) as run_info_file:
                 model["run_info"] = yaml.load(run_info_file, yaml.Loader)
         else:
             if model._clean and is_dir_non_empty(model_dir):
                 shutil.rmtree(model_dir)
-            zip_ref.extractall(model.working_dir)
+            zip_ref.extractall(os.path.join(model.working_dir, str(model.id)))
 
             # set model_dir
             model.run_info["model_dir"] = model_dir