Delete file cache to complete public API

This removes the final dependency we had that didn't go through c-ray.h
- The file cache.
It was a hack I devised a while ago to facilitate network rendering, and
it allowed me to avoid doing what this patch does, which is to write
proper JSON serializing logic for most internal state.
That's now done (for now, it lives in protocol/protocol.c), and
simplifies many aspects of setting up and syncing with nodes.

To make it possible to serialize everything, numerous other improvements
popped up - Textures and shader graphs are now just arrays that live in
the scene object, instead of the complicated refcounting I had there
before. Meshes and instances just refer to those with indices now.
This also resolves the bug I recently introduced where textures got
loaded twice for no reason. Textures are also no longer borrowing the
node graph memory pool.

The API got some changes too, those material sets and vertex buffers now
get controlled with handles instead of internally refcounted pointers,
since they just live in a shared array.

I'd imagine that some of the serialize/deserialize logic could be
combined with json_loader, didn't really pay attention to that idea yet.

Massive patch, but most of these changes really can't be separated out
to multiple smaller steps - The old file cache did touch many things.

I added a bunch of FIXMEs and TODOs again. The milestone of actually
separating the project into a shared object + driver program is getting
really close now! Really happy with this commit :^)

include/c-ray/c-ray.h

@@ -46,7 +46,6 @@ enum cr_renderer_param {
 	cr_renderer_should_save, //FIXME: Remove
 	cr_renderer_override_cam,
 	cr_renderer_node_list,
-	cr_renderer_scene_cache, // FIXME: Remove
 	cr_renderer_is_iterative,
 };
 
@@ -136,7 +135,7 @@ typedef cr_object cr_sphere;
 cr_sphere cr_scene_add_sphere(struct cr_scene *s_ext, float radius);
 typedef cr_object cr_mesh;
 
-struct cr_vertex_buf {
+struct cr_vertex_buf_param {
 	struct cr_vector *vertices;
 	size_t vertex_count;
 	struct cr_vector *normals;
@@ -145,9 +144,9 @@ struct cr_vertex_buf {
 	size_t tex_coord_count;
 };
 
-struct cr_vertex_buf *cr_vertex_buf_new(struct cr_vertex_buf in);
-void cr_vertex_buf_del(struct cr_vertex_buf *buf);
-void cr_mesh_bind_vertex_buf(struct cr_scene *s_ext, cr_mesh mesh, struct cr_vertex_buf *buf);
+typedef cr_object cr_vertex_buf;
+cr_vertex_buf cr_scene_vertex_buf_new(struct cr_scene *s_ext, struct cr_vertex_buf_param in);
+void cr_mesh_bind_vertex_buf(struct cr_scene *s_ext, cr_mesh mesh, cr_vertex_buf buf);
 
 struct cr_face {
 	int vertex_idx[MAX_CRAY_VERTEX_COUNT];
@@ -189,10 +188,9 @@ bool cr_camera_update(struct cr_scene *ext, cr_camera c);
 // -- Materials --
 #include "node.h"
 
-typedef struct cr_material_set cr_material_set;
-cr_material_set *cr_material_set_new(void);
-void cr_material_set_add(struct cr_renderer *r_ext, cr_material_set *set, struct cr_shader_node *desc);
-void cr_material_set_del(cr_material_set *set);
+typedef cr_object cr_material_set;
+cr_material_set cr_scene_new_material_set(struct cr_scene *s_ext);
+void cr_material_set_add(struct cr_renderer *r_ext, struct cr_scene *s_ext, cr_material_set set, struct cr_shader_node *desc);
 
 // -- Instancing --
 
@@ -204,7 +202,7 @@ enum cr_object_type {
 
 cr_instance cr_instance_new(struct cr_scene *s_ext, cr_object object, enum cr_object_type type);
 void cr_instance_set_transform(struct cr_scene *s_ext, cr_instance instance, float row_major[4][4]);
-bool cr_instance_bind_material_set(struct cr_renderer *r_ext, cr_instance instance, cr_material_set *set);
+bool cr_instance_bind_material_set(struct cr_renderer *r_ext, cr_instance instance, cr_material_set set);
 
 // -- Misc. --
 bool cr_scene_set_background(struct cr_renderer *r_ext, struct cr_scene *s_ext, struct cr_shader_node *desc);

include/c-ray/node.h

@@ -236,6 +236,7 @@ struct cr_vector_node {
 };
 
 struct cr_shader_node {
+	// TODO: s/bsdf/shader
 	enum cr_bsdf_node_type {
 		cr_bsdf_unknown = 0,
 		cr_bsdf_diffuse,

src/api/c-ray.c

@@ -24,10 +24,11 @@
 #include "../utils/string.h"
 #include "../utils/protocol/server.h"
 #include "../utils/protocol/worker.h"
-#include "../utils/filecache.h"
 #include "../utils/hashtable.h"
 #include "../datatypes/camera.h"
 #include "../utils/loaders/textureloader.h"
+#include "../driver/node_parse.h"
+#include "../utils/protocol/protocol.h"
 
 #ifdef CRAY_DEBUG_ENABLED
 #define DEBUG "D"
@@ -132,8 +133,9 @@ bool cr_renderer_set_str_pref(struct cr_renderer *ext, enum cr_renderer_param p,
 			return true;
 		}
 		case cr_renderer_asset_path: {
-			if (r->prefs.assetPath) free(r->prefs.assetPath);
-			r->prefs.assetPath = stringCopy(str);
+			// TODO: we shouldn't really be touching anything but prefs in here.
+			if (r->scene->asset_path) free(r->scene->asset_path);
+			r->scene->asset_path = stringCopy(str);
 			return true;
 		}
 		case cr_renderer_output_name: {
@@ -156,12 +158,6 @@ bool cr_renderer_set_str_pref(struct cr_renderer *ext, enum cr_renderer_param p,
 		case cr_renderer_node_list: {
 			if (r->prefs.node_list) free(r->prefs.node_list);
 			r->prefs.node_list = stringCopy(str);
-			if (!r->state.file_cache) r->state.file_cache = cache_create();
-			return true;
-		}
-		case cr_renderer_scene_cache: {
-			if (r->sceneCache) free(r->sceneCache);
-			r->sceneCache = stringCopy(str);
 			return true;
 		}
 		default: {
@@ -201,7 +197,7 @@ const char *cr_renderer_get_str_pref(struct cr_renderer *ext, enum cr_renderer_p
 	switch (p) {
 		case cr_renderer_output_path: return r->prefs.imgFilePath;
 		case cr_renderer_output_name: return r->prefs.imgFileName;
-		case cr_renderer_asset_path: return r->prefs.assetPath;
+		case cr_renderer_asset_path: return r->scene->asset_path;
 		default: return NULL;
 	}
 	return NULL;
@@ -226,10 +222,12 @@ uint64_t cr_renderer_get_num_pref(struct cr_renderer *ext, enum cr_renderer_para
 	return 0;
 }
 
+// TODO: Remove r_ext
 bool cr_scene_set_background(struct cr_renderer *r_ext, struct cr_scene *s_ext, struct cr_shader_node *desc) {
 	if (!r_ext || !s_ext) return false;
 	struct world *s = (struct world *)s_ext;
-	s->background = desc ? build_bsdf_node(r_ext, desc) : newBackground(&s->storage, NULL, NULL, NULL);
+	s->background = desc ? build_bsdf_node(s_ext, desc) : newBackground(&s->storage, NULL, NULL, NULL);
+	s->bg_desc = desc ? desc : NULL;
 	return true;
 }
 
@@ -259,37 +257,37 @@ cr_sphere cr_scene_add_sphere(struct cr_scene *s_ext, float radius) {
 	return sphere_arr_add(&scene->spheres, (struct sphere){ .radius = radius });
 }
 
-struct cr_vertex_buf *cr_vertex_buf_new(struct cr_vertex_buf in) {
-	struct vertex_buffer *buf = vertex_buf_ref(NULL);
+cr_vertex_buf cr_scene_vertex_buf_new(struct cr_scene *s_ext, struct cr_vertex_buf_param in) {
+	if (!s_ext) return -1;
+	struct world *scene = (struct world *)s_ext;
+	struct vertex_buffer new = { 0 };
 	// TODO: T_arr_add_n()
 	if (in.vertices && in.vertex_count) {
 		for (size_t i = 0; i < in.vertex_count; ++i) {
-			vector_arr_add(&buf->vertices, *(struct vector *)&in.vertices[i]);
+			vector_arr_add(&new.vertices, *(struct vector *)&in.vertices[i]);
 		}
 	}
 	if (in.normals && in.normal_count) {
 		for (size_t i = 0; i < in.normal_count; ++i) {
-			vector_arr_add(&buf->normals, *(struct vector *)&in.normals[i]);
+			vector_arr_add(&new.normals, *(struct vector *)&in.normals[i]);
 		}
 	}
 	if (in.tex_coords && in.tex_coord_count) {
 		for (size_t i = 0; i < in.tex_coord_count; ++i) {
-			coord_arr_add(&buf->texture_coords, *(struct coord *)&in.tex_coords[i]);
+			coord_arr_add(&new.texture_coords, *(struct coord *)&in.tex_coords[i]);
 		}
 	}
-	return (struct cr_vertex_buf *)buf;
+	return vertex_buffer_arr_add(&scene->v_buffers, new);
 }
 
-void cr_vertex_buf_del(struct cr_vertex_buf *buf) {
-	vertex_buf_unref((struct vertex_buffer *)buf);
-}
-
-void cr_mesh_bind_vertex_buf(struct cr_scene *s_ext, cr_mesh mesh, struct cr_vertex_buf *buf) {
-	if (!s_ext || !buf) return;
+void cr_mesh_bind_vertex_buf(struct cr_scene *s_ext, cr_mesh mesh, cr_vertex_buf buf) {
+	if (!s_ext) return;
 	struct world *scene = (struct world *)s_ext;
 	if ((size_t)mesh > scene->meshes.count - 1) return;
 	struct mesh *m = &scene->meshes.items[mesh];
-	m->vbuf = vertex_buf_ref((struct vertex_buffer *)buf);
+	if ((size_t)buf > scene->v_buffers.count - 1) return;
+	m->vbuf = &scene->v_buffers.items[buf];
+	m->vbuf_idx = buf;
 }
 
 void cr_mesh_bind_faces(struct cr_scene *s_ext, cr_mesh mesh, struct cr_face *faces, size_t face_count) {
@@ -348,13 +346,15 @@ void cr_instance_set_transform(struct cr_scene *s_ext, cr_instance instance, flo
 	};
 }
 
-bool cr_instance_bind_material_set(struct cr_renderer *r_ext, cr_instance instance, cr_material_set *set) {
-	if (!r_ext || !set) return false;
+bool cr_instance_bind_material_set(struct cr_renderer *r_ext, cr_instance instance, cr_material_set set) {
+	if (!r_ext) return false;
 	struct renderer *r = (struct renderer *)r_ext;
 	struct world *scene = r->scene;
 	if ((size_t)instance > scene->instances.count - 1) return false;
+	if ((size_t)set > scene->shader_buffers.count - 1) return false;
 	struct instance *i = &scene->instances.items[instance];
-	i->bbuf = bsdf_buf_ref((struct bsdf_buffer *)set);
+	i->bbuf = &scene->shader_buffers.items[set];
+	i->bbuf_idx = set;
 	return true;
 }
 
@@ -459,29 +459,29 @@ bool cr_camera_remove(struct cr_scene *s, cr_camera c) {
 
 // --
 
-cr_material_set *cr_material_set_new(void) {
-	return (cr_material_set *)bsdf_buf_ref(NULL);
+cr_material_set cr_scene_new_material_set(struct cr_scene *s_ext) {
+	if (!s_ext) return -1;
+	struct world *scene = (struct world *)s_ext;
+	return bsdf_buffer_arr_add(&scene->shader_buffers, (struct bsdf_buffer){ 0 });
 }
 
-void cr_material_set_add(struct cr_renderer *r_ext, cr_material_set *set, struct cr_shader_node *desc) {
-	if (!set || !desc) return;
-	struct bsdf_buffer *buf = (struct bsdf_buffer *)set;
-	const struct bsdfNode *node = build_bsdf_node(r_ext, desc);
+// TODO: Remove r_ext
+void cr_material_set_add(struct cr_renderer *r_ext, struct cr_scene *s_ext, cr_material_set set, struct cr_shader_node *desc) {
+	if (!r_ext || !s_ext) return;
+	struct renderer *r = (struct renderer *)r_ext;
+	struct world *s = (struct world *)s_ext;
+	if (s != r->scene) return;
+	if ((size_t)set > s->shader_buffers.count - 1) return;
+	struct bsdf_buffer *buf = &s->shader_buffers.items[set];
+	const struct bsdfNode *node = build_bsdf_node(s_ext, desc);
+	cr_shader_node_ptr_arr_add(&buf->descriptions, desc);
 	bsdf_node_ptr_arr_add(&buf->bsdfs, node);
 }
 
-void cr_material_set_del(cr_material_set *set) {
-	if (!set) return;
-	bsdf_buf_unref((struct bsdf_buffer *)set);
-}
-
 struct texture *cr_renderer_render(struct cr_renderer *ext) {
 	struct renderer *r = (struct renderer *)ext;
 	if (r->prefs.node_list) {
 		r->state.clients = clients_sync(r);
-		free(r->sceneCache);
-		r->sceneCache = NULL;
-		cache_destroy(r->state.file_cache);
 	}
 	if (!r->state.clients.count && !r->prefs.threads) {
 		logr(warning, "You specified 0 local threads, and no network clients were found. Nothing to do.\n");

src/datatypes/image/texture.h

@@ -11,6 +11,7 @@
 #include <stdbool.h>
 #include <stddef.h>
 #include "../color.h"
+#include "../../utils/dyn_array.h"
 
 enum colorspace {
 	linear,
@@ -35,6 +36,14 @@ struct texture {
 	size_t height;
 };
 
+struct texture_asset {
+	char *path;
+	struct texture *t;
+};
+
+typedef struct texture_asset texture_asset;
+dyn_array_def(texture_asset);
+
 struct texture *newTexture(enum precision p, size_t width, size_t height, size_t channels);
 
 void setPixel(struct texture *t, struct color c, size_t x, size_t y);

src/datatypes/mesh.c

@@ -17,27 +17,13 @@
 void destroyMesh(struct mesh *mesh) {
 	if (mesh) {
 		free(mesh->name);
-		vertex_buf_unref(mesh->vbuf);
 		poly_arr_free(&mesh->polygons);
 		destroy_bvh(mesh->bvh);
 	}
 }
 
-struct vertex_buffer *vertex_buf_ref(struct vertex_buffer *buf) {
-	if (buf) {
-		buf->refs++;
-		return buf;
-	}
-	struct vertex_buffer *new = calloc(1, sizeof(*new));
-	new->refs = 1;
-	return new;
-}
-
-void vertex_buf_unref(struct vertex_buffer *buf) {
-	if (!buf) return;
-	if (--buf->refs) return;
-	vector_arr_free(&buf->vertices);
-	vector_arr_free(&buf->normals);
-	coord_arr_free(&buf->texture_coords);
-	free(buf);
+void vertex_buf_free(struct vertex_buffer buf) {
+	vector_arr_free(&buf.vertices);
+	vector_arr_free(&buf.normals);
+	coord_arr_free(&buf.texture_coords);
 }

src/datatypes/mesh.h

@@ -13,22 +13,20 @@
 #include "poly.h"
 #include "material.h"
 
-/*
-	Several meshes may point to the same vertex buffer, shared among meshes in
-	a wavefront file, for instance. A simple refcount is kept to make it easier
-	to automatically free it when adding and removing meshes.
-*/
 struct vertex_buffer {
 	struct vector_arr vertices;
 	struct vector_arr normals;
 	struct coord_arr texture_coords;
-	size_t refs;
 };
 
+typedef struct vertex_buffer vertex_buffer;
+dyn_array_def(vertex_buffer);
+
 struct mesh {
 	struct vertex_buffer *vbuf;
 	struct poly_arr polygons;
 	struct bvh *bvh;
+	size_t vbuf_idx;
 	float surface_area;
 	char *name;
 	float rayOffset;
@@ -39,5 +37,4 @@ dyn_array_def(mesh);
 
 void destroyMesh(struct mesh *mesh);
 
-struct vertex_buffer *vertex_buf_ref(struct vertex_buffer *buf);
-void vertex_buf_unref(struct vertex_buffer *buf);
+void vertex_buf_free(struct vertex_buffer buf);

src/datatypes/scene.c

@@ -13,14 +13,22 @@
 #include "../utils/hashtable.h"
 #include "../utils/textbuffer.h"
 #include "../utils/dyn_array.h"
+#include "../driver/node_parse.h"
+#include "image/texture.h"
 #include "camera.h"
 #include "tile.h"
 #include "mesh.h"
 #include "poly.h"
 
-//Free scene data
+void tex_asset_free(struct texture_asset *a) {
+	if (a->path) free(a->path);
+	if (a->t) destroyTexture(a->t);
+}
+
 void scene_destroy(struct world *scene) {
 	if (scene) {
+		scene->textures.elem_free = tex_asset_free;
+		texture_asset_arr_free(&scene->textures);
 		camera_arr_free(&scene->cameras);
 		for (size_t i = 0; i < scene->meshes.count; ++i) {
 			destroyMesh(&scene->meshes.items[i]);
@@ -29,11 +37,23 @@ void scene_destroy(struct world *scene) {
 		destroy_bvh(scene->topLevel);
 		destroyHashtable(scene->storage.node_table);
 		destroyBlocks(scene->storage.node_pool);
-		for (size_t i = 0; i < scene->instances.count; ++i) {
-			bsdf_buf_unref(scene->instances.items[i].bbuf);
+		for (size_t i = 0; i < scene->shader_buffers.count; ++i) {
+			for (size_t j = 0; j < scene->shader_buffers.items[i].descriptions.count; ++j) {
+				cr_shader_node_free(scene->shader_buffers.items[i].descriptions.items[j]);
+			}
+			cr_shader_node_ptr_arr_free(&scene->shader_buffers.items[i].descriptions);
+			bsdf_node_ptr_arr_free(&scene->shader_buffers.items[i].bsdfs);
+		}
+		bsdf_buffer_arr_free(&scene->shader_buffers);
+		cr_shader_node_free(scene->bg_desc);
+		// TODO: set as dyn_array elem_free somewhere
+		for (size_t i = 0; i < scene->v_buffers.count; ++i) {
+			vertex_buf_free(scene->v_buffers.items[i]);
 		}
+		vertex_buffer_arr_free(&scene->v_buffers);
 		instance_arr_free(&scene->instances);
 		sphere_arr_free(&scene->spheres);
+		if (scene->asset_path) free(scene->asset_path);
 		free(scene);
 	}
 }