From 930dc374651b57ddde465488c1a8b232bde4c895 Mon Sep 17 00:00:00 2001 From: Valtteri Koskivuori Date: Mon, 13 Nov 2023 23:24:03 +0200 Subject: [PATCH] More rage-hacking, ripping out things and putting them in main() Most overrides are now passed in from main, which is very good. Soon we can rewrite the args parser to not have that awful global state. main.c is getting awfully cluttered though, but that will be resolved over time. We just have to trust the process. I'm also moving pretty fast, so I'm likely adding lots of bugs, I'll work on those once I've got the API and lib/driver separation to a satisfactory state. --- include/c-ray/c-ray.h | 2 + src/accelerators/bvh.c | 45 ++++++++++ src/accelerators/bvh.h | 2 + src/api/c-ray.c | 13 ++- src/main.c | 70 ++++++++++++++- src/renderer/renderer.c | 38 +++++++++ src/renderer/renderer.h | 2 +- src/utils/args.c | 2 +- src/utils/loaders/sceneloader.c | 147 -------------------------------- src/utils/protocol/server.c | 18 ++-- src/utils/protocol/server.h | 2 +- src/utils/protocol/worker.c | 25 ++++++ 12 files changed, 202 insertions(+), 164 deletions(-) diff --git a/include/c-ray/c-ray.h b/include/c-ray/c-ray.h index 4e33bdf1..89da409e 100644 --- a/include/c-ray/c-ray.h +++ b/include/c-ray/c-ray.h @@ -42,6 +42,8 @@ enum cr_renderer_param { cr_renderer_override_height, cr_renderer_should_save, //FIXME: Remove cr_renderer_override_cam, + cr_renderer_node_list, + cr_renderer_scene_cache, // FIXME: Remove }; bool cr_renderer_set_num_pref(struct cr_renderer *ext, enum cr_renderer_param p, uint64_t num); diff --git a/src/accelerators/bvh.c b/src/accelerators/bvh.c index 811d179c..437b1ce2 100644 --- a/src/accelerators/bvh.c +++ b/src/accelerators/bvh.c @@ -15,6 +15,9 @@ #include "../datatypes/mesh.h" #include "../renderer/instance.h" #include "../datatypes/vector.h" +#include "../utils/platform/thread.h" +#include "../utils/platform/signal.h" +#include "../utils/timer.h" #include #include @@ -643,3 +646,45 @@ void destroy_bvh(struct bvh *bvh) { free(bvh); } } + +struct bvh_build_task { + struct bvh *bvh; + const struct mesh *mesh; +}; + +void *bvh_build_thread(void *arg) { + block_signals(); + struct bvh_build_task *task = (struct bvh_build_task *)thread_user_data(arg); + task->bvh = build_mesh_bvh(task->mesh); + return NULL; +} + +void compute_accels(struct mesh_arr meshes) { + logr(info, "Computing BVHs: "); + struct timeval timer = { 0 }; + timer_start(&timer); + struct bvh_build_task *tasks = calloc(meshes.count, sizeof(*tasks)); + struct cr_thread *build_threads = calloc(meshes.count, sizeof(*build_threads)); + for (size_t t = 0; t < meshes.count; ++t) { + tasks[t] = (struct bvh_build_task){ + .mesh = &meshes.items[t], + }; + build_threads[t] = (struct cr_thread){ + .thread_fn = bvh_build_thread, + .user_data = &tasks[t] + }; + if (thread_start(&build_threads[t])) { + logr(error, "Failed to create a bvhBuildTask\n"); + } + } + + for (size_t t = 0; t < meshes.count; ++t) { + thread_wait(&build_threads[t]); + meshes.items[t].bvh = tasks[t].bvh; + } + printSmartTime(timer_get_ms(timer)); + free(tasks); + free(build_threads); + logr(plain, "\n"); +} + diff --git a/src/accelerators/bvh.h b/src/accelerators/bvh.h index 0b6a6318..9d563e90 100644 --- a/src/accelerators/bvh.h +++ b/src/accelerators/bvh.h @@ -51,3 +51,5 @@ bool traverse_bottom_level_bvh( /// Frees the memory allocated by the given BVH void destroy_bvh(struct bvh *); + +void compute_accels(struct mesh_arr meshes); diff --git a/src/api/c-ray.c b/src/api/c-ray.c index a9e7e99d..162455c2 100644 --- a/src/api/c-ray.c +++ b/src/api/c-ray.c @@ -151,6 +151,16 @@ bool cr_renderer_set_str_pref(struct cr_renderer *ext, enum cr_renderer_param p, } return true; } + case cr_renderer_node_list: { + if (r->prefs.node_list) free(r->prefs.node_list); + r->prefs.node_list = stringCopy(str); + return true; + } + case cr_renderer_scene_cache: { + if (r->sceneCache) free(r->sceneCache); + r->sceneCache = stringCopy(str); + return true; + } default: { logr(warning, "Renderer param %i not a string\n", p); } @@ -356,8 +366,7 @@ void cr_load_mesh_from_buf(char *buf) { struct texture *cr_renderer_render(struct cr_renderer *ext) { struct renderer *r = (struct renderer *)ext; - if (args_is_set("use_clustering")) { - r->prefs.useClustering = true; + if (r->prefs.node_list) { r->state.clients = syncWithClients(r, &r->state.clientCount); free(r->sceneCache); r->sceneCache = NULL; diff --git a/src/main.c b/src/main.c index f573f784..b36ca5d4 100644 --- a/src/main.c +++ b/src/main.c @@ -66,6 +66,10 @@ int main(int argc, char *argv[]) { goto done; } + if (args_is_set("cam_index")) { + cr_renderer_set_num_pref(renderer, cr_renderer_override_cam, args_int("cam_index")); + } + // FIXME: Remove global options table, store it in a local in main() and run overrides // from there. @@ -114,9 +118,71 @@ int main(int argc, char *argv[]) { } } - if (args_is_set("cam_index")) { - cr_renderer_set_num_pref(renderer, cr_renderer_override_cam, args_int("cam_index")); + // This is where we prepare a cache of scene data to be sent to worker nodes + // We also apply any potential command-line overrides to that cache here as well. + // FIXME: This overrides setting should be integrated with scene loading, probably. + if (args_is_set("nodes_list")) { + cr_renderer_set_str_pref(renderer, cr_renderer_node_list, args_string("nodes_list")); + // Stash a cache of scene data here + // Apply overrides to the cache here + if (args_is_set("samples_override")) { + cJSON *renderer = cJSON_GetObjectItem(scene, "renderer"); + if (cJSON_IsObject(renderer)) { + int samples = args_int("samples_override"); + logr(debug, "Overriding cache sample count to %i\n", samples); + if (cJSON_IsNumber(cJSON_GetObjectItem(renderer, "samples"))) { + cJSON_ReplaceItemInObject(renderer, "samples", cJSON_CreateNumber(samples)); + } else { + cJSON_AddItemToObject(renderer, "samples", cJSON_CreateNumber(samples)); + } + } + } + + if (args_is_set("dims_override")) { + cJSON *renderer = cJSON_GetObjectItem(scene, "renderer"); + if (cJSON_IsObject(renderer)) { + int width = args_int("dims_width"); + int height = args_int("dims_height"); + logr(info, "Overriding cache image dimensions to %ix%i\n", width, height); + if (cJSON_IsNumber(cJSON_GetObjectItem(renderer, "width")) && cJSON_IsNumber(cJSON_GetObjectItem(renderer, "height"))) { + cJSON_ReplaceItemInObject(renderer, "width", cJSON_CreateNumber(width)); + cJSON_ReplaceItemInObject(renderer, "height", cJSON_CreateNumber(height)); + } else { + cJSON_AddItemToObject(renderer, "width", cJSON_CreateNumber(width)); + cJSON_AddItemToObject(renderer, "height", cJSON_CreateNumber(height)); + } + } + } + + if (args_is_set("tiledims_override")) { + cJSON *renderer = cJSON_GetObjectItem(scene, "renderer"); + if (cJSON_IsObject(renderer)) { + int width = args_int("tile_width"); + int height = args_int("tile_height"); + logr(info, "Overriding cache tile dimensions to %ix%i\n", width, height); + if (cJSON_IsNumber(cJSON_GetObjectItem(renderer, "tileWidth")) && cJSON_IsNumber(cJSON_GetObjectItem(renderer, "tileHeight"))) { + cJSON_ReplaceItemInObject(renderer, "tileWidth", cJSON_CreateNumber(width)); + cJSON_ReplaceItemInObject(renderer, "tileHeight", cJSON_CreateNumber(height)); + } else { + cJSON_AddItemToObject(renderer, "tileWidth", cJSON_CreateNumber(width)); + cJSON_AddItemToObject(renderer, "tileHeight", cJSON_CreateNumber(height)); + } + } + } + + if (args_is_set("cam_index")) { + cJSON_AddItemToObject(scene, "selected_camera", cJSON_CreateNumber(args_int("cam_index"))); + } + + // Store cache. This is what gets sent to worker nodes. + char *cache = cJSON_PrintUnformatted(scene); + cr_renderer_set_str_pref(renderer, cr_renderer_scene_cache, cache); + free(cache); } + + logr(debug, "Deleting JSON...\n"); + cJSON_Delete(scene); + logr(debug, "Deleting done\n"); struct timeval timer; timer_start(&timer); diff --git a/src/renderer/renderer.c b/src/renderer/renderer.c index 8189ea79..23675466 100644 --- a/src/renderer/renderer.c +++ b/src/renderer/renderer.c @@ -29,6 +29,7 @@ #include "../utils/platform/signal.h" #include "../utils/protocol/server.h" #include "../utils/string.h" +#include "../accelerators/bvh.h" //Main thread loop speeds #define paused_msec 100 @@ -44,6 +45,29 @@ void sigHandler(int sig) { } } +static void printSceneStats(struct world *scene, unsigned long long ms) { + logr(info, "Scene construction completed in "); + printSmartTime(ms); + uint64_t polys = 0; + uint64_t vertices = 0; + uint64_t normals = 0; + for (size_t i = 0; i < scene->instances.count; ++i) { + if (isMesh(&scene->instances.items[i])) { + const struct mesh *mesh = &scene->meshes.items[scene->instances.items[i].object_idx]; + polys += mesh->polygons.count; + vertices += mesh->vbuf->vertices.count; + normals += mesh->vbuf->normals.count; + } + } + logr(plain, "\n"); + logr(info, "Totals: %liV, %liN, %zuI, %liP, %zuS, %zuM\n", + vertices, + normals, + scene->instances.count, + polys, + scene->spheres.count, + scene->meshes.count); +} void *renderThread(void *arg); void *renderThreadInteractive(void *arg); @@ -92,6 +116,20 @@ struct texture *renderFrame(struct renderer *r) { r->prefs.tileHeight, r->prefs.tileOrder); + // Do some pre-render preparations + // Compute BVH acceleration structures for all meshes in the scene + compute_accels(r->scene->meshes); + + // And then compute a single top-level BVH that contains all the objects + logr(info, "Computing top-level BVH: "); + struct timeval timer = {0}; + timer_start(&timer); + r->scene->topLevel = build_top_level_bvh(r->scene->instances); + printSmartTime(timer_get_ms(timer)); + logr(plain, "\n"); + + printSceneStats(r->scene, timer_get_ms(timer)); + for (size_t i = 0; i < r->state.tiles.count; ++i) r->state.tiles.items[i].total_samples = r->prefs.sampleCount; diff --git a/src/renderer/renderer.h b/src/renderer/renderer.h index a79ff795..9cdd4fc1 100644 --- a/src/renderer/renderer.h +++ b/src/renderer/renderer.h @@ -77,7 +77,7 @@ struct prefs { char *assetPath; size_t imgCount; enum fileType imgType; - bool useClustering; + char *node_list; bool isWorker; struct sdl_prefs window; }; diff --git a/src/utils/args.c b/src/utils/args.c index 66af87f9..b9fb22eb 100644 --- a/src/utils/args.c +++ b/src/utils/args.c @@ -240,7 +240,7 @@ void args_parse(int argc, char **argv) { logr(debug, "Verbose mode enabled\n"); if (args_is_set("shutdown") && args_is_set("nodes_list")) { - shutdownClients(); + shutdownClients(args_string("nodes_list")); term_restore(); exit(0); } diff --git a/src/utils/loaders/sceneloader.c b/src/utils/loaders/sceneloader.c index 1f2dec02..1a8f756c 100644 --- a/src/utils/loaders/sceneloader.c +++ b/src/utils/loaders/sceneloader.c @@ -39,7 +39,6 @@ #include "../../utils/timer.h" #include "../../utils/string.h" #include "../../utils/platform/signal.h" -#include "../../accelerators/bvh.h" #include "../../nodes/bsdfnode.h" #include "../../nodes/valuenode.h" #include "../../nodes/colornode.h" @@ -690,71 +689,6 @@ static void parseScene(struct cr_renderer *r, const cJSON *data) { parse_meshes(todo_remove_r, cJSON_GetObjectItem(data, "meshes")); } -struct bvh_build_task { - struct bvh *bvh; - const struct mesh *mesh; -}; - -void *bvh_build_thread(void *arg) { - block_signals(); - struct bvh_build_task *task = (struct bvh_build_task *)thread_user_data(arg); - task->bvh = build_mesh_bvh(task->mesh); - return NULL; -} - -static void compute_accels(struct mesh_arr meshes) { - logr(info, "Computing BVHs: "); - struct timeval timer = { 0 }; - timer_start(&timer); - struct bvh_build_task *tasks = calloc(meshes.count, sizeof(*tasks)); - struct cr_thread *build_threads = calloc(meshes.count, sizeof(*build_threads)); - for (size_t t = 0; t < meshes.count; ++t) { - tasks[t] = (struct bvh_build_task){ - .mesh = &meshes.items[t], - }; - build_threads[t] = (struct cr_thread){ - .thread_fn = bvh_build_thread, - .user_data = &tasks[t] - }; - if (thread_start(&build_threads[t])) { - logr(error, "Failed to create a bvhBuildTask\n"); - } - } - - for (size_t t = 0; t < meshes.count; ++t) { - thread_wait(&build_threads[t]); - meshes.items[t].bvh = tasks[t].bvh; - } - printSmartTime(timer_get_ms(timer)); - free(tasks); - free(build_threads); - logr(plain, "\n"); -} - -static void printSceneStats(struct world *scene, unsigned long long ms) { - logr(info, "Scene construction completed in "); - printSmartTime(ms); - uint64_t polys = 0; - uint64_t vertices = 0; - uint64_t normals = 0; - for (size_t i = 0; i < scene->instances.count; ++i) { - if (isMesh(&scene->instances.items[i])) { - const struct mesh *mesh = &scene->meshes.items[scene->instances.items[i].object_idx]; - polys += mesh->polygons.count; - vertices += mesh->vbuf->vertices.count; - normals += mesh->vbuf->normals.count; - } - } - logr(plain, "\n"); - logr(info, "Totals: %liV, %liN, %zuI, %liP, %zuS, %zuM\n", - vertices, - normals, - scene->instances.count, - polys, - scene->spheres.count, - scene->meshes.count); -} - int parse_json(struct cr_renderer *r, cJSON *json) { struct timeval timer = {0}; timer_start(&timer); @@ -785,86 +719,5 @@ int parse_json(struct cr_renderer *r, cJSON *json) { // -------------- - // This is where we prepare a cache of scene data to be sent to worker nodes - // We also apply any potential command-line overrides to that cache here as well. - // FIXME: This overrides setting should be integrated with scene loading, probably. - if (args_is_set("use_clustering")) { - // Stash a cache of scene data here - // Apply overrides to the cache here - if (args_is_set("samples_override")) { - cJSON *renderer = cJSON_GetObjectItem(json, "renderer"); - if (cJSON_IsObject(renderer)) { - int samples = args_int("samples_override"); - logr(debug, "Overriding cache sample count to %i\n", samples); - if (cJSON_IsNumber(cJSON_GetObjectItem(renderer, "samples"))) { - cJSON_ReplaceItemInObject(renderer, "samples", cJSON_CreateNumber(samples)); - } else { - cJSON_AddItemToObject(renderer, "samples", cJSON_CreateNumber(samples)); - } - } - } - - if (args_is_set("dims_override")) { - cJSON *renderer = cJSON_GetObjectItem(json, "renderer"); - if (cJSON_IsObject(renderer)) { - int width = args_int("dims_width"); - int height = args_int("dims_height"); - logr(info, "Overriding cache image dimensions to %ix%i\n", width, height); - if (cJSON_IsNumber(cJSON_GetObjectItem(renderer, "width")) && cJSON_IsNumber(cJSON_GetObjectItem(renderer, "height"))) { - cJSON_ReplaceItemInObject(renderer, "width", cJSON_CreateNumber(width)); - cJSON_ReplaceItemInObject(renderer, "height", cJSON_CreateNumber(height)); - } else { - cJSON_AddItemToObject(renderer, "width", cJSON_CreateNumber(width)); - cJSON_AddItemToObject(renderer, "height", cJSON_CreateNumber(height)); - } - } - } - - if (args_is_set("tiledims_override")) { - cJSON *renderer = cJSON_GetObjectItem(json, "renderer"); - if (cJSON_IsObject(renderer)) { - int width = args_int("tile_width"); - int height = args_int("tile_height"); - logr(info, "Overriding cache tile dimensions to %ix%i\n", width, height); - if (cJSON_IsNumber(cJSON_GetObjectItem(renderer, "tileWidth")) && cJSON_IsNumber(cJSON_GetObjectItem(renderer, "tileHeight"))) { - cJSON_ReplaceItemInObject(renderer, "tileWidth", cJSON_CreateNumber(width)); - cJSON_ReplaceItemInObject(renderer, "tileHeight", cJSON_CreateNumber(height)); - } else { - cJSON_AddItemToObject(renderer, "tileWidth", cJSON_CreateNumber(width)); - cJSON_AddItemToObject(renderer, "tileHeight", cJSON_CreateNumber(height)); - } - } - } - - if (todo_remove_r->prefs.selected_camera != 0) { - cJSON_AddItemToObject(json, "selected_camera", cJSON_CreateNumber(todo_remove_r->prefs.selected_camera)); - } - - // Store cache. This is what gets sent to worker nodes. - todo_remove_r->sceneCache = cJSON_PrintUnformatted(json); - } - - logr(debug, "Deleting JSON...\n"); - cJSON_Delete(json); - logr(debug, "Deleting done\n"); - - if (todo_remove_r->prefs.threads > 0) { - // Do some pre-render preparations - // Compute BVH acceleration structures for all meshes in the scene - compute_accels(todo_remove_r->scene->meshes); - - // And then compute a single top-level BVH that contains all the objects - logr(info, "Computing top-level BVH: "); - struct timeval timer = {0}; - timer_start(&timer); - todo_remove_r->scene->topLevel = build_top_level_bvh(todo_remove_r->scene->instances); - printSmartTime(timer_get_ms(timer)); - logr(plain, "\n"); - - printSceneStats(todo_remove_r->scene, timer_get_ms(timer)); - } else { - logr(debug, "No local render threads, skipping local BVH construction.\n"); - } - return 0; } diff --git a/src/utils/protocol/server.c b/src/utils/protocol/server.c index 50fd37fc..1ff864f8 100644 --- a/src/utils/protocol/server.c +++ b/src/utils/protocol/server.c @@ -105,17 +105,15 @@ static bool connectToClient(struct renderClient *client) { return success; } -// Fetches list of nodes from arguments, verifies that they are reachable, and -// returns them in a nice list. Also got the size there in the amount param, if you need it. -static struct renderClient *buildClientList(size_t *amount) { - ASSERT(args_is_set("use_clustering")); - ASSERT(args_is_set("nodes_list")); - char *nodesString = args_string("nodes_list"); +// Fetches list of nodes from node string, verifies that they are reachable, and +// returns them in a nice list. +static struct renderClient *buildClientList(const char *node_list, size_t *amount) { + ASSERT(node_list); // Really barebones parsing for IP addresses and ports in a comma-separated list // Expected to break easily. Don't break it. char buf[LINEBUFFER_MAXSIZE]; lineBuffer line = { .buf = buf }; - fillLineBuffer(&line, nodesString, ','); + fillLineBuffer(&line, node_list, ','); ASSERT(line.amountOf.tokens > 0); size_t clientCount = line.amountOf.tokens; struct renderClient *clients = calloc(clientCount, sizeof(*clients)); @@ -352,9 +350,9 @@ static void *client_sync_thread(void *arg) { return NULL; } -void shutdownClients() { +void shutdownClients(const char *node_list) { size_t clientCount = 0; - struct renderClient *clients = buildClientList(&clientCount); + struct renderClient *clients = buildClientList(node_list, &clientCount); logr(info, "Sending shutdown command to %zu client%s.\n", clientCount, PLURAL(clientCount)); if (clientCount < 1) { logr(warning, "No clients found, exiting\n"); @@ -396,7 +394,7 @@ void printProgressBars(struct syncThreadParams *params, size_t clientCount) { struct renderClient *syncWithClients(const struct renderer *r, size_t *count) { signal(SIGPIPE, SIG_IGN); size_t clientCount = 0; - struct renderClient *clients = buildClientList(&clientCount); + struct renderClient *clients = buildClientList(r->prefs.node_list, &clientCount); if (clientCount < 1) { logr(warning, "No clients found, rendering solo.\n"); if (count) *count = 0; diff --git a/src/utils/protocol/server.h b/src/utils/protocol/server.h index 856da721..9352a42f 100644 --- a/src/utils/protocol/server.h +++ b/src/utils/protocol/server.h @@ -35,7 +35,7 @@ struct renderClient { int id; }; -void shutdownClients(void); +void shutdownClients(const char *node_list); // Synchronise renderer state with clients, and return a list of clients // ready to do some rendering diff --git a/src/utils/protocol/worker.c b/src/utils/protocol/worker.c index 0d97414d..af721ed1 100644 --- a/src/utils/protocol/worker.c +++ b/src/utils/protocol/worker.c @@ -34,6 +34,7 @@ #include "../timer.h" #include "../args.h" #include "../../utils/platform/signal.h" +#include "../../accelerators/bvh.h" #include #include @@ -233,7 +234,18 @@ static cJSON *startRender(int connectionSocket) { struct workerThreadState *workerThreadStates = calloc(threadCount, sizeof(*workerThreadStates)); struct camera selected_cam = g_worker_renderer->scene->cameras.items[g_worker_renderer->prefs.selected_camera]; + if (g_worker_renderer->prefs.override_width && g_worker_renderer->prefs.override_height) { + selected_cam.width = g_worker_renderer->prefs.override_width ? (int)g_worker_renderer->prefs.override_width : selected_cam.width; + selected_cam.height = g_worker_renderer->prefs.override_height ? (int)g_worker_renderer->prefs.override_height : selected_cam.height; + cam_recompute_optics(&selected_cam); + } struct renderer *r = g_worker_renderer; + logr(info, "Got job: %s%i%s x %s%i%s, %s%zu%s samples with %s%zu%s bounces\n", KWHT, selected_cam.width, KNRM, KWHT, selected_cam.height, KNRM, KBLU, r->prefs.sampleCount, KNRM, KGRN, r->prefs.bounces, KNRM); + logr(info, "Rendering with %s%zu%s local thread%s.\n", + KRED, + r->prefs.threads, + KNRM, + PLURAL(r->prefs.threads)); //Quantize image into renderTiles tile_quantize(&r->state.tiles, selected_cam.width, @@ -242,6 +254,19 @@ static cJSON *startRender(int connectionSocket) { r->prefs.tileHeight, r->prefs.tileOrder); + logr(info, "%u x %u tiles\n", r->prefs.tileWidth, r->prefs.tileHeight); + // Do some pre-render preparations + // Compute BVH acceleration structures for all meshes in the scene + compute_accels(r->scene->meshes); + + // And then compute a single top-level BVH that contains all the objects + logr(info, "Computing top-level BVH: "); + struct timeval timer = {0}; + timer_start(&timer); + r->scene->topLevel = build_top_level_bvh(r->scene->instances); + printSmartTime(timer_get_ms(timer)); + logr(plain, "\n"); + for (size_t i = 0; i < r->state.tiles.count; ++i) r->state.tiles.items[i].total_samples = r->prefs.sampleCount;