Avoid initializing LibFS twice (issue #5) + adding more assertions.

README.md

@@ -192,7 +192,7 @@ For more advanced configurations, we describe additional parameters.
     * -DDIGEST_OPT: Enable log coalescing during digestion.
 
 * **Both Makefiles**
-    * -DISTRIBUTED: Use Assise in distributed mode (otherwise, Assise runs as a single-process local filesystem).
+    * -DISTRIBUTED: Use Assise in distributed mode (always keep this enabled; to-be-removed in the future).
     * -USE_LEASE : Enable leases for LibFS/KernFS.
     * -USE_SSD: Enable usage of an SSD.
 
@@ -206,7 +206,7 @@ For more advanced configurations, we describe additional parameters.
     * g_n_max_libfs: Maximum number of LibFS processes allowed to run simultaneously.
     * g_max_read_cache_blocks: Maximum number of read cache blocks.
 
-**Note:** g_n_max_libfs * g_log_size should be <= dev_size[g_log_dev].
+**Note:** (g_n_max_libfs + g_n_hot_rep) * g_log_size should be <= dev_size[g_log_dev].
 
 ## Troubleshooting ##
 

libfs/src/distributed/peer.c

@@ -233,12 +233,14 @@ void register_peer_log(struct peer_id *peer, int find_id)
 	if(find_id) {
 		idx = find_first_zero_bit(peer_bitmap, peer_bitmap_size);
 		mlfs_debug("peer id: %u bitmap: %lu\n", idx, peer_bitmap[0]);
-		print_bits(peer_bitmap[0]);
+		//print_bits(peer_bitmap[0]);
 	}
 	else {
 		idx = peer->id;
 	}
 
+	if(idx >= g_n_max_libfs + g_n_nodes)
+		panic("Unable to register LibFS log. Try increasing g_n_max_libfs\n");
 
 	mlfs_info("assigning peer (ip: %s pid: %u) to log id %d\n", peer->ip, peer->pid, idx);
 	peer->id = idx;

libfs/src/distributed/replication.c

@@ -18,7 +18,7 @@ int g_rsync_rf = g_n_nodes;	//default: replicate to all nodes
 int log_idx = 0;
 
 #ifdef KERNFS
-struct replication_context *g_sync_ctx[g_n_max_libfs];
+struct replication_context *g_sync_ctx[g_n_max_libfs + g_n_nodes];
 #else
 struct replication_context *g_sync_ctx[g_n_nodes];
 #endif

libfs/src/distributed/replication.h

@@ -13,7 +13,7 @@
 #define N_RSYNC_THREADS 2
 
 #ifdef KERNFS
-extern struct replication_context *g_sync_ctx[g_n_max_libfs];
+extern struct replication_context *g_sync_ctx[g_n_max_libfs + g_n_nodes];
 #else
 extern struct replication_context *g_sync_ctx[g_n_nodes];
 #endif

libfs/src/filesystem/fs.c

@@ -151,7 +151,7 @@ void shutdown_fs(void)
 	int ret;
 	int _enable_perf_stats = enable_perf_stats;
 
-	if (!initialized) {
+	if(!cmpxchg(&initialized, 1, 0)) {
 		return ;
 	}
 
@@ -188,8 +188,6 @@ void shutdown_fs(void)
 		panic("cannot close shared memory\n");
 	*/
 
-	initialized = 0;
-
 	return ;
 }
 
@@ -409,12 +407,10 @@ void init_fs(void)
 	sigaction(SIGINT, &action, NULL);
 	sigaction(SIGTERM, &action, NULL);
 
-	if (!initialized) {
+	if(!cmpxchg(&initialized, 0, 1)) {
 		const char *perf_profile;
 		int i;
 
-		initialized = 1;
-
 #ifdef USE_SLAB
 		mlfs_slab_init(memsize_gb << 30);
 #endif
@@ -475,6 +471,8 @@ void init_fs(void)
 		clock_speed_mhz = get_cpu_clock_speed();
 
 	}
+	else
+		mlfs_printf("LibFS already initialized. Skipping..%s\n", "");
 }
 
 ///////////////////////////////////////////////////////////////////////

libfs/src/mkfs/mkfs.c

@@ -156,11 +156,11 @@ int main(int argc, char *argv[])
 	}
 
 	// Bypass dev-dax mmap problem: use actual device size - 550 MB.
-	file_size_bytes = dev_size[dev_id] - (550 << 20);
+	file_size_bytes = dev_size[dev_id] - (4 << 20);
 	file_size_blks = file_size_bytes >> g_block_size_shift;
 
 	if(dev_id == g_log_dev) {
-		log_size_blks = g_log_size * g_n_max_libfs + (1UL * (1 << 10)); 
+		log_size_blks = g_log_size * (g_n_max_libfs + g_n_nodes) + (1UL * (1 << 10));
 		mlfs_assert(file_size_blks > log_size_blks);
 		file_size_blks -= log_size_blks;
 	}

libfs/tests/Makefile

@@ -45,9 +45,6 @@ small_io: small_io.c
 append_test: append_test.c
 	$(CC) -g -o $@ $^ -I$(INCLUDES) -L$(LIBFS_DIR) -lmlfs -L$(RDMA_CORE_DIR) -L$(RDMA_DIR) $(LDFLAGS)
 
-mkdir_user: mkdir_user.c
-	$(CC) -g -o $@ $^ -I$(INCLUDES) -L$(LIBFS_DIR) -lmlfs -L$(RDMA_CORE_DIR) -L$(RDMA_DIR) $(LDFLAGS)
-
 signal_test: signal_test.c
 	$(CC) -g -o $@ $^ -I$(INCLUDES) -L$(LIBFS_DIR) -lmlfs -L$(RDMA_CORE_DIR) -L$(RDMA_DIR) $(LDFLAGS)
 
@@ -81,9 +78,6 @@ maildir_test: maildir_test.c time_stat.o
 mkdir_test: mkdir_test.c time_stat.o
 	$(CC) -g -o $@ $^ -I$(INCLUDES) -L$(LIBFS_DIR) -lmlfs -DMLFS $(CFLAGS) -L$(RDMA_CORE_DIR) -L$(RDMA_DIR) $(LDFLAGS)
 
-write_read: write_read.c time_stat.o
-	$(CC) -g -o $(addsuffix .normal, $@) $^ $(CFLAGS) $(DAX_OBJ) -L$(NVML_DIR) -L$(RDMA_CORE_DIR) -L$(RDMA_DIR) -lpmem -lpthread -lm -lrt  -Wl,-rpath=$(abspath $(NVML_DIR))
-
 iotest: iotest.cc time_stat.o thread.cc
 	$(CXX) -std=c++11 -g -Ofast -o $@ $^  -I$(INCLUDES) -L$(LIBFS_DIR) -lmlfs  -L$(RDMA_CORE_DIR) -L$(RDMA_DIR) -DMLFS $(CFLAGS) $(LDFLAGS)
 	$(CXX) -std=c++11 -g -Ofast -o $(addsuffix .normal, $@) $^ $(CFLAGS) $(DAX_OBJ) -L$(NVML_DIR) -lpmem -lpthread -lm -lrt -Wl,-rpath=$(abspath $(NVML_DIR))

libfs/tests/ftrunc_test.c

@@ -25,7 +25,7 @@ int main(int argc, char ** argv)
 	unsigned long file_size;
 	struct stat statbuf;
 
-	init_fs();
+	//init_fs();
 
 	ret = mkdir("/mlfs/", 0600);
 
@@ -100,7 +100,7 @@ int main(int argc, char ** argv)
 
 	unlink(FILE_NAME);
 
-	shutdown_fs();
+	//shutdown_fs();
 
 	return 0;
 }

libfs/tests/many_files_test.cc

@@ -17,7 +17,9 @@
 #include <string>
 #include <vector>
 
+#ifdef MLFS
 #include "mlfs/mlfs_interface.h"
+#endif
 
 #include "time_stat.h"
 #include "thread.h"
@@ -67,7 +69,7 @@ void io_bench::prepare(void)
 {
 	int ret;
 
-	init_fs();
+	//init_fs();
 
 	ret = mkdir(test_dir_prefix, 0777);
 
@@ -376,7 +378,7 @@ int main(int argc, char *argv[])
 	fflush(stdout);
 	fflush(stderr);
 
-	shutdown_fs();
+	//shutdown_fs();
 
 	return 0;
 }

libfs/tests/mkdir_user.c

@@ -6,12 +6,11 @@
 
 #include <mlfs/mlfs_interface.h>	
 
-
 int main(int argc, char ** argv)
 {
-	struct stat st = {0};
+    struct stat st = {0};
 
-	init_fs();
+    //init_fs();
 
     char const* const fileName = "userlist.dat"; /* should check that argc > 1 */
     char const* mlfs_dir = "/mlfs/";
@@ -42,6 +41,7 @@ int main(int argc, char ** argv)
 
     fclose(file);
 
-	shutdown_fs();
-	return 0;
+    //shutdown_fs();
+
+    return 0;
 }

libfs/tests/readdir_test.c

@@ -1,6 +1,7 @@
 #include <stdio.h>
 #include <dirent.h>
 #include <fcntl.h>
+
 #include <mlfs/mlfs_interface.h>
 
 int main(int argc, char ** argv)
@@ -9,7 +10,8 @@ int main(int argc, char ** argv)
 
     DIR * dir;
 
-    init_fs();
+    //init_fs();
+
     dir = opendir("/mlfs");
 
 	//dir_test creates 1000 files in /mlfs/test_dir/files
@@ -20,5 +22,7 @@ int main(int argc, char ** argv)
 
     closedir(dir);
 
+    //shutdown_fs();
+
     return 0;
 }