Unformatted (binary) checkpointing

.github/workflows/backend-template.yml

@@ -184,3 +184,12 @@ jobs:
           name: failed-interface-test-diffs-${{ inputs.backend_name }}
           path: ${{ inputs.test_interface_directory }}/*/*.diff
 
+      - name: Run regression tests for library code with ${{ inputs.backend_name }} (binary restarts)
+        if: ${{ inputs.test_lib_directory }}
+        shell: bash
+        working-directory: ${{ inputs.test_lib_directory }}
+        env:
+          COLVARS_BINARY_RESTART: 1
+        run: |
+          apptainer exec ${{github.workspace}}/devel-tools/${{ inputs.container_name }}.sif \
+          ./run_tests.sh ${{github.workspace}}/${{ inputs.backend_name }}-source/${{ inputs.rpath_exe }} 0??_*

cmake/CMakeLists.txt

@@ -142,7 +142,7 @@ if(COLVARS_DEBUG)
 endif()
 
 option(ENABLE_COVERAGE "Enable code coverage reporting" ON)
-if(ENABLE_COVERAGE AND (NOT CMAKE_CXX_COMPILER_ID STREQUAL "SunPro"))
+if(ENABLE_COVERAGE AND (NOT CMAKE_CXX_COMPILER_ID STREQUAL "SunPro") AND (NOT CMAKE_CXX_COMPILER_ID STREQUAL "MSVC"))
   target_compile_options(colvars PUBLIC --coverage)
   target_link_options(colvars PUBLIC --coverage)
 endif()

colvartools/poisson_integrator.cpp

@@ -1,3 +1,4 @@
+#include <iostream>
 #include <fstream>
 
 #include "colvargrid.h"

gromacs/tests/library/run_tests.sh

@@ -223,6 +223,12 @@ for dir in ${DIRLIST} ; do
   for f in AutoDiff/*
   do
     base=`basename $f`
+
+    if [ "${base%.state.stripped}" != "${base}" ] && [ -n "${COLVARS_BINARY_RESTART}" ] ; then
+      # Do not try comparing binary state files, they will never match anyway
+      continue
+    fi
+
     if [ ! -f $base ] ; then
       echo -e "\n*** File $(${TPUT_RED})$base$(${TPUT_CLEAR}) is missing. ***"
       SUCCESS=0

lammps/doc/src/fix_colvars.rst

@@ -85,9 +85,10 @@ selection keywords in the Colvars configuration file or files.  There is
 no need to define multiple "fix colvars" instances and it is not
 allowed.
 
-The "output" keyword allows to specify the prefix of output files
-generated by Colvars, for example "*output*.colvars.traj" or
-"output.pmf".
+The "output" keyword allows to specify the prefix of output files generated
+by Colvars, for example "*output*.colvars.traj" or "output.pmf".  Supplying
+an empty string suppresses any file output from Colvars to file, except for
+data saved into the LAMMPS :doc:`binary restart <restart>` files.
 
 The "input" keyword allows to specify an optional state file that contains
 the restart information needed to continue a previous simulation state.

lammps/lib/colvars/Makefile.common

@@ -59,6 +59,7 @@ COLVARS_SRCS = \
         colvarscript_commands.cpp \
         colvarscript_commands_bias.cpp \
         colvarscript_commands_colvar.cpp \
+        colvars_memstream.cpp \
         colvartypes.cpp \
         colvarvalue.cpp \
         colvar_neuralnetworkcompute.cpp

lammps/src/COLVARS/colvarproxy_lammps.cpp

@@ -206,23 +206,6 @@ double colvarproxy_lammps::compute()
   return bias_energy;
 }
 
-void colvarproxy_lammps::serialize_status(std::string &rst)
-{
-  std::ostringstream os;
-  colvars->write_restart(os);
-  rst = os.str();
-}
-
-// set status from string
-bool colvarproxy_lammps::deserialize_status(std::string &rst)
-{
-  if (! colvarproxy_io::input_stream_from_string("input state string", rst)) {
-    return false;
-  } else {
-    return true;
-  }
-}
-
 
 cvm::rvector colvarproxy_lammps::position_distance(cvm::atom_pos const &pos1,
                                                    cvm::atom_pos const &pos2)

lammps/src/COLVARS/colvarproxy_lammps.h

@@ -76,12 +76,6 @@ class colvarproxy_lammps : public colvarproxy {
   // perform colvars computation. returns biasing energy
   double compute();
 
-  // dump status to string
-  void serialize_status(std::string &);
-
-  // set status from string
-  bool deserialize_status(std::string &);
-
   // Request to set the units used internally by Colvars
   int set_unit_system(std::string const &units_in, bool check_only = false) override;
 

lammps/src/COLVARS/fix_colvars.cpp

@@ -45,9 +45,11 @@
 #include <iostream>
 #include <vector>
 
-#include "colvarproxy_lammps.h"
 #include "colvarmodule.h"
+#include "colvarproxy.h"
+#include "colvarproxy_lammps.h"
 #include "colvarscript.h"
+#include "colvars_memstream.h"
 
 
 /* struct for packed data communication of coordinates and forces. */
@@ -888,13 +890,17 @@ void FixColvars::end_of_step()
 void FixColvars::write_restart(FILE *fp)
 {
   if (me == 0) {
-    std::string rest_text;
-    proxy->serialize_status(rest_text);
-    // TODO call write_output_files()
-    const char *cvm_state = rest_text.c_str();
-    int len = strlen(cvm_state) + 1; // need to include terminating null byte.
-    fwrite(&len,sizeof(int),1,fp);
-    fwrite(cvm_state,1,len,fp);
+    cvm::memory_stream ms;
+    if (proxy->colvars->write_state(ms)) {
+      int len_cv_state = ms.length();
+      // Will write the buffer's length twice, so that the fix can read it later, too
+      int len = len_cv_state + sizeof(int);
+      fwrite(&len, sizeof(int), 1, fp);
+      fwrite(&len, sizeof(int), 1, fp);
+      fwrite(ms.output_buffer(), 1, len_cv_state, fp);
+    } else {
+      error->all(FLERR, "Failed to write Colvars state to binary file");
+    }
   }
 }
 
@@ -903,8 +909,10 @@ void FixColvars::write_restart(FILE *fp)
 void FixColvars::restart(char *buf)
 {
   if (me == 0) {
-    std::string rest_text(buf);
-    if (!proxy->deserialize_status(rest_text)) {
+    // Read the buffer's length, then load it into Colvars starting right past that location
+    int length = *(reinterpret_cast<int *>(buf));
+    unsigned char *colvars_state_buffer = reinterpret_cast<unsigned char *>(buf + sizeof(int));
+    if (proxy->colvars->set_input_state_buffer(length, colvars_state_buffer) != COLVARS_OK) {
       error->all(FLERR, "Failed to set the Colvars input state from string buffer");
     }
   }

lammps/tests/library/run_tests.sh

@@ -229,6 +229,10 @@ for dir in ${DIRLIST} ; do
   for f in AutoDiff/*
   do
     base=`basename $f`
+    if [ "${base%.state.stripped}" != "${base}" ] && [ -n "${COLVARS_BINARY_RESTART}" ] ; then
+      # Do not try comparing binary state files, they will never match anyway
+      continue
+    fi
     if [ "${base}" != "${base%.traj}" ] ; then
       # System force is now total force
       sed 's/fs_/ft_/g' < ${base} > ${TMPDIR}/${base}

misc_interfaces/stubs/colvarproxy_stub.cpp

@@ -7,6 +7,8 @@
 // If you wish to distribute your changes, please submit them to the
 // Colvars repository at GitHub.
 
+#include <iostream>
+
 #include "colvarmodule.h"
 #include "colvarscript.h"
 #include "colvaratoms.h"

namd/colvars/Make.depends

@@ -7,40 +7,41 @@ obj/colvar.o: \
 	colvars/src/colvartypes.h \
 	colvars/src/colvarparse.h \
 	colvars/src/colvarparams.h \
-	colvars/src/colvar.h \
-	colvars/src/colvardeps.h \
 	colvars/src/colvarcomp.h \
 	colvars/src/colvaratoms.h \
 	colvars/src/colvarproxy.h \
 	colvars/src/colvarproxy_io.h \
 	colvars/src/colvarproxy_system.h \
 	colvars/src/colvarproxy_tcl.h \
 	colvars/src/colvarproxy_volmaps.h \
+	colvars/src/colvardeps.h \
+	colvars/src/colvar.h \
 	colvars/src/colvar_arithmeticpath.h \
 	colvars/src/colvar_geometricpath.h \
 	colvars/src/colvarscript.h \
 	colvars/src/colvarbias.h \
 	colvars/src/colvarscript_commands.h \
 	colvars/src/colvarscript_commands_colvar.h \
-	colvars/src/colvarscript_commands_bias.h
+	colvars/src/colvarscript_commands_bias.h \
+	colvars/src/colvars_memstream.h
 	$(CXX) $(COLVARSCXXFLAGS) $(COPTO)obj/colvar.o $(COPTC) colvars/src/colvar.cpp
 obj/colvaratoms.o: \
 	obj/.exists \
 	colvars/src/colvaratoms.cpp \
+	colvars/src/colvardeps.h \
 	colvars/src/colvarmodule.h \
 	colvars/src/colvars_version.h \
-	colvars/src/colvarproxy.h \
-	colvars/src/colvartypes.h \
+	colvars/src/colvarparse.h \
 	colvars/src/colvarvalue.h \
+	colvars/src/colvartypes.h \
+	colvars/src/colvarparams.h \
+	colvars/src/colvarproxy.h \
 	colvars/src/colvarproxy_io.h \
 	colvars/src/colvarproxy_system.h \
 	colvars/src/colvarproxy_tcl.h \
 	colvars/src/colvarproxy_volmaps.h \
-	colvars/src/colvarparse.h \
-	colvars/src/colvarparams.h \
 	colvars/src/colvaratoms.h \
-	colvars/src/colvardeps.h \
-        colvars/src/colvar_rotation_derivative.h
+	colvars/src/colvar_rotation_derivative.h
 	$(CXX) $(COLVARSCXXFLAGS) $(COPTO)obj/colvaratoms.o $(COPTC) colvars/src/colvaratoms.cpp
 obj/colvarbias.o: \
 	obj/.exists \
@@ -87,10 +88,15 @@ obj/colvarbias_alb.o: \
 	colvars/src/colvarbias_alb.cpp \
 	colvars/src/colvarmodule.h \
 	colvars/src/colvars_version.h \
+	colvars/src/colvarproxy.h \
+	colvars/src/colvartypes.h \
+	colvars/src/colvarvalue.h \
+	colvars/src/colvarproxy_io.h \
+	colvars/src/colvarproxy_system.h \
+	colvars/src/colvarproxy_tcl.h \
+	colvars/src/colvarproxy_volmaps.h \
 	colvars/src/colvarbias.h \
 	colvars/src/colvar.h \
-	colvars/src/colvarvalue.h \
-	colvars/src/colvartypes.h \
 	colvars/src/colvarparse.h \
 	colvars/src/colvarparams.h \
 	colvars/src/colvardeps.h \
@@ -301,7 +307,7 @@ obj/colvarcomp_distances.o: \
 	colvars/src/colvarproxy_volmaps.h \
 	colvars/src/colvar_arithmeticpath.h \
 	colvars/src/colvar_geometricpath.h \
-        colvars/src/colvar_rotation_derivative.h
+	colvars/src/colvar_rotation_derivative.h
 	$(CXX) $(COLVARSCXXFLAGS) $(COPTO)obj/colvarcomp_distances.o $(COPTC) colvars/src/colvarcomp_distances.cpp
 obj/colvarcomp_gpath.o: \
 	obj/.exists \
@@ -365,7 +371,7 @@ obj/colvarcomp_rotations.o: \
 	colvars/src/colvarproxy_volmaps.h \
 	colvars/src/colvar_arithmeticpath.h \
 	colvars/src/colvar_geometricpath.h \
-        colvars/src/colvar_rotation_derivative.h
+	colvars/src/colvar_rotation_derivative.h
 	$(CXX) $(COLVARSCXXFLAGS) $(COPTO)obj/colvarcomp_rotations.o $(COPTC) colvars/src/colvarcomp_rotations.cpp
 obj/colvarcomp_volmaps.o: \
 	obj/.exists \
@@ -394,18 +400,18 @@ obj/colvarcomp_combination.o: \
 	colvars/src/colvarcomp.h \
 	colvars/src/colvarmodule.h \
 	colvars/src/colvars_version.h \
-	colvars/src/colvar.h \
-	colvars/src/colvarvalue.h \
-	colvars/src/colvartypes.h \
-	colvars/src/colvarparse.h \
-	colvars/src/colvarparams.h \
-	colvars/src/colvardeps.h \
 	colvars/src/colvaratoms.h \
 	colvars/src/colvarproxy.h \
+	colvars/src/colvartypes.h \
+	colvars/src/colvarvalue.h \
 	colvars/src/colvarproxy_io.h \
 	colvars/src/colvarproxy_system.h \
 	colvars/src/colvarproxy_tcl.h \
 	colvars/src/colvarproxy_volmaps.h \
+	colvars/src/colvarparse.h \
+	colvars/src/colvarparams.h \
+	colvars/src/colvardeps.h \
+	colvars/src/colvar.h \
 	colvars/src/colvar_arithmeticpath.h \
 	colvars/src/colvar_geometricpath.h
 	$(CXX) $(COLVARSCXXFLAGS) $(COPTO)obj/colvarcomp_combination.o $(COPTC) colvars/src/colvarcomp_combination.cpp
@@ -538,7 +544,8 @@ obj/colvarparse.o: \
 	colvars/src/colvarvalue.h \
 	colvars/src/colvartypes.h \
 	colvars/src/colvarparse.h \
-	colvars/src/colvarparams.h
+	colvars/src/colvarparams.h \
+	colvars/src/colvars_memstream.h
 	$(CXX) $(COLVARSCXXFLAGS) $(COPTO)obj/colvarparse.o $(COPTC) colvars/src/colvarparse.cpp
 obj/colvarproxy.o: \
 	obj/.exists \
@@ -705,6 +712,15 @@ obj/colvarscript_commands_colvar.o: \
 	colvars/src/colvarscript_commands_colvar.h \
 	colvars/src/colvarscript_commands_bias.h
 	$(CXX) $(COLVARSCXXFLAGS) $(COPTO)obj/colvarscript_commands_colvar.o $(COPTC) colvars/src/colvarscript_commands_colvar.cpp
+obj/colvars_memstream.o: \
+	obj/.exists \
+	colvars/src/colvars_memstream.cpp \
+	colvars/src/colvarmodule.h \
+	colvars/src/colvars_version.h \
+	colvars/src/colvartypes.h \
+	colvars/src/colvarvalue.h \
+	colvars/src/colvars_memstream.h
+	$(CXX) $(COLVARSCXXFLAGS) $(COPTO)obj/colvars_memstream.o $(COPTC) colvars/src/colvars_memstream.cpp
 obj/colvartypes.o: \
 	obj/.exists \
 	colvars/src/colvartypes.cpp \
@@ -714,16 +730,24 @@ obj/colvartypes.o: \
 	colvars/src/colvarparse.h \
 	colvars/src/colvarvalue.h \
 	colvars/src/colvarparams.h \
-	colvars/src/nr_jacobi.h \
-        colvars/src/colvar_rotation_derivative.h
+	colvars/src/colvaratoms.h \
+	colvars/src/colvarproxy.h \
+	colvars/src/colvarproxy_io.h \
+	colvars/src/colvarproxy_system.h \
+	colvars/src/colvarproxy_tcl.h \
+	colvars/src/colvarproxy_volmaps.h \
+	colvars/src/colvardeps.h \
+	colvars/src/colvar_rotation_derivative.h \
+	colvars/src/nr_jacobi.h
 	$(CXX) $(COLVARSCXXFLAGS) $(COPTO)obj/colvartypes.o $(COPTC) colvars/src/colvartypes.cpp
 obj/colvarvalue.o: \
 	obj/.exists \
 	colvars/src/colvarvalue.cpp \
 	colvars/src/colvarmodule.h \
 	colvars/src/colvars_version.h \
 	colvars/src/colvarvalue.h \
-	colvars/src/colvartypes.h
+	colvars/src/colvartypes.h \
+	colvars/src/colvars_memstream.h
 	$(CXX) $(COLVARSCXXFLAGS) $(COPTO)obj/colvarvalue.o $(COPTC) colvars/src/colvarvalue.cpp
 obj/nr_jacobi.o: \
 	obj/.exists \

namd/colvars/src/Makefile.namd

@@ -36,6 +36,7 @@ COLVARSLIB = \
 	$(DSTDIR)/colvarscript_commands.o \
 	$(DSTDIR)/colvarscript_commands_bias.o \
 	$(DSTDIR)/colvarscript_commands_colvar.o \
+	$(DSTDIR)/colvars_memstream.o \
 	$(DSTDIR)/colvartypes.o \
 	$(DSTDIR)/colvarvalue.o \
 	$(DSTDIR)/nr_jacobi.o

namd/src/colvarproxy_namd.C

@@ -1049,7 +1049,7 @@ std::ostream & colvarproxy_namd::output_stream(std::string const &output_name,
 
   backup_file(output_name.c_str());
 
-  output_streams_[output_name] = new ofstream_namd(output_name.c_str());
+  output_streams_[output_name] = new ofstream_namd(output_name.c_str(), std::ios::binary);
   if (! output_streams_[output_name]->good()) {
     cvm::error("Error: cannot write to "+description+" \""+output_name+"\".\n",
                COLVARS_FILE_ERROR);

namd/tests/library/run_tests.sh

@@ -161,15 +161,18 @@ for dir in ${DIRLIST} ; do
     script=`basename ${script}`
     basename=${script%.namd}
 
-    # Don't do multithreading for reinitatoms
-    NUM_THREADS_THIS=${NUM_THREADS}
-    if [ ${dir} == 003_reinitatoms ] ; then
-      NUM_THREADS_THIS=1
+    # If we are doing binary restarts, make an exception for tests that don't support it
+    if [ -n "${COLVARS_BINARY_RESTART}" ] ; then
+      if [ ${dir} == 004_10ala_moving_restart ] ; then
+        export COLVARS_BINARY_RESTART=0
+      else
+        export COLVARS_BINARY_RESTART=1
+      fi
     fi
 
     # Run the test (use a subshell to avoid cluttering stdout)
     # Use multiple threads to test SMP code (TODO: move SMP tests to interface?)
-    NAMD_CUDASOA=$CUDASOA $BINARY +p ${NUM_THREADS_THIS} $script > ${basename}.out
+    NAMD_CUDASOA=$CUDASOA $BINARY +p ${NUM_THREADS} $script > ${basename}.out
 
     # Output of Colvars module, minus the version numbers
     grep "^colvars:" ${basename}.out | grep -v 'Initializing the collective variables module' \
@@ -227,6 +230,10 @@ for dir in ${DIRLIST} ; do
   for f in AutoDiff/*
   do
     base=`basename $f`
+    if [ "${base%.state.stripped}" != "${base}" ] && [ -n "${COLVARS_BINARY_RESTART}" ] ; then
+      # Do not try comparing binary state files, they will never match anyway
+      continue
+    fi
     if [ ! -f $base ] ; then
       echo -e "\n*** File $(${TPUT_RED})$base$(${TPUT_CLEAR}) is missing. ***"
       SUCCESS=0

src/.clang-format

@@ -8,4 +8,7 @@ ObjCBlockIndentWidth: 2
 PenaltyBreakAssignment: 2
 AccessModifierOffset: -2
 BreakBeforeBraces: WebKit
+EmptyLineAfterAccessModifier: Leave
+NamespaceIndentation: None
+MaxEmptyLinesToKeep: 2
 ...