io: Various changes wrt IO

IO is now more prepared to be asynchronous. Some headers have been pulled apart. Signed-off-by: Pedro Falcato <[email protected]>
heatd · Jan 5, 2024 · 79b495d · 79b495d
1 parent d72c715
commit 79b495d
Show file tree

Hide file tree

Showing 24 changed files with 758 additions and 236 deletions.
diff --git a/kernel/arch/x86_64/multiboot2.cpp b/kernel/arch/x86_64/multiboot2.cpp
@@ -16,7 +16,6 @@
 
 #include <onyx/acpi.h>
 #include <onyx/binfmt.h>
-#include <onyx/block.h>
 #include <onyx/bootmem.h>
 #include <onyx/clock.h>
 #include <onyx/cmdline.h>

diff --git a/kernel/drivers/ahci/ahci.cpp b/kernel/drivers/ahci/ahci.cpp
@@ -372,31 +372,29 @@ long ahci_setup_prdt_bio(prdt_t *prdt, struct bio_req *r, size_t *size)
 
 bool ahci_do_command(struct ahci_port *ahci_port, struct ahci_command_ata *buf)
 {
-    struct bio_req *r = bio_alloc_and_init(GFP_KERNEL);
+    struct bio_req *r = bio_alloc(GFP_KERNEL, buf->nr_iov);
     if (!r)
         return false;
 
     r->bdev = ahci_port->bdev.get();
-    r->curr_vec_index = 0;
     r->flags = BIO_REQ_DEVICE_SPECIFIC;
     r->device_specific[0] = buf->cmd;
     r->sector_number = 0;
-    r->nr_vecs = buf->nr_iov;
-    r->vec = buf->iovec;
+    memcpy(r->vec, buf->iovec, buf->nr_iov * sizeof(struct page_iov));
 
     if (ahci_port->io_queue->submit_request(r) < 0)
         return false;
 
     int st = wait_for(
-        &r,
+        r,
         [](void *_req) -> bool {
-            struct bio_req *r = (struct bio_req *) _req;
-            return r->flags & (BIO_REQ_DONE | BIO_REQ_EIO);
+            struct bio_req *req = (struct bio_req *) _req;
+            return req->flags & (BIO_REQ_DONE | BIO_REQ_EIO);
         },
         WAIT_FOR_FOREVER, 0);
 
     bool ret = st == 0 && !(r->flags & BIO_REQ_EIO);
-    bio_free(r);
+    bio_put(r);
     return ret;
 }
 

diff --git a/kernel/include/onyx/bdev_base_types.h b/kernel/include/onyx/bdev_base_types.h
@@ -0,0 +1,53 @@
+/*
+ * Copyright (c) 2016 - 2023 Pedro Falcato
+ * This file is part of Onyx, and is released under the terms of the MIT License
+ * check LICENSE at the root directory for more information
+ *
+ * SPDX-License-Identifier: MIT
+ */
+#ifndef _ONYX_BDEV_BASE_TYPES_H
+#define _ONYX_BDEV_BASE_TYPES_H
+
+#include <onyx/list.h>
+#include <onyx/page_iov.h>
+#include <onyx/types.h>
+
+/* Keep basic bdev types that are universally used and exported to consumers here. Do not keep
+ * blockdev nor io_queue here.
+ */
+
+struct blockdev;
+struct io_queue;
+
+typedef u64 sector_t;
+
+#define BIO_REQ_OP_MASK         (0xff)
+#define BIO_REQ_READ_OP         0
+#define BIO_REQ_WRITE_OP        1
+#define BIO_REQ_DEVICE_SPECIFIC 2
+
+/* BIO flags start at bit 8 since bits 0 - 7 are reserved for operations */
+/* Note that we still have 24 bits for flags, which should be More Than Enough(tm) */
+#define BIO_REQ_DONE     (1 << 8)
+#define BIO_REQ_EIO      (1 << 9)
+#define BIO_REQ_TIMEOUT  (1 << 10)
+#define BIO_REQ_NOT_SUPP (1 << 11)
+
+struct bio_req
+{
+    unsigned int b_ref;
+    uint32_t flags;
+    sector_t sector_number;
+    struct page_iov *vec;
+    size_t nr_vecs;
+    size_t curr_vec_index;
+    struct blockdev *bdev;
+    struct io_queue *b_queue;
+    struct list_head list_node;
+    unsigned long device_specific[4];
+    void (*b_end_io)(struct bio_req *req);
+    void *b_private;
+    struct page_iov b_inline_vec[];
+};
+
+#endif
diff --git a/kernel/include/onyx/bio.h b/kernel/include/onyx/bio.h
@@ -0,0 +1,74 @@
+/*
+ * Copyright (c) 2016 - 2023 Pedro Falcato
+ * This file is part of Onyx, and is released under the terms of the MIT License
+ * check LICENSE at the root directory for more information
+ *
+ * SPDX-License-Identifier: MIT
+ */
+#ifndef _ONYX_BIO_H
+#define _ONYX_BIO_H
+
+#include <onyx/bdev_base_types.h>
+#include <onyx/compiler.h>
+
+#define BIO_MAX_INLINE_VECS 8
+
+/**
+ * @brief Allocate a bio_req
+ * The system will attempt to allocate a bio_req with an inline page_iov vector. If not possible, it
+ * will allocate them on the heap.
+ *
+ * @param gfp_flags GFP flags
+ * @param nr_vecs Number of vectors
+ * @return The allocated, initialized bio_req
+ */
+struct bio_req *bio_alloc(unsigned int gfp_flags, size_t nr_vectors);
+
+/**
+ * @brief Free a bio_req
+ *
+ * @param req Request to free
+ */
+void bio_free(struct bio_req *req);
+
+static inline void bio_init(struct bio_req *req)
+{
+    *req = {};
+    req->b_ref = 1;
+}
+
+static inline void bio_get(struct bio_req *req)
+{
+    __atomic_add_fetch(&req->b_ref, 1, __ATOMIC_ACQUIRE);
+}
+
+static inline void bio_put(struct bio_req *req)
+{
+    if (__atomic_sub_fetch(&req->b_ref, 1, __ATOMIC_RELEASE) == 0)
+        bio_free(req);
+}
+
+/**
+ * @brief Submit a bio_req and wait for it to end
+ *
+ * @param dev Block device
+ * @param req Request
+ * @return errno-like result of the bio_req
+ */
+int bio_submit_req_wait(struct blockdev *dev, struct bio_req *req);
+
+/**
+ * @brief Complete a bio req
+ * Callable from softirq context, and lower
+ *
+ * @param req Request to complete
+ */
+static inline void bio_do_complete(struct bio_req *req)
+{
+    req->flags |= BIO_REQ_DONE;
+    if (req->b_end_io)
+        req->b_end_io(req);
+    bio_put(req);
+}
+
+#endif
diff --git a/kernel/include/onyx/block.h b/kernel/include/onyx/block.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016 - 2023 Pedro Falcato
+ * Copyright (c) 2016 - 2024 Pedro Falcato
  * This file is part of Onyx, and is released under the terms of the MIT License
  * check LICENSE at the root directory for more information
  *
@@ -12,12 +12,13 @@
 #include <stdint.h>
 #include <stdlib.h>
 
+#include <onyx/bdev_base_types.h>
+#include <onyx/bio.h>
 #include <onyx/culstring.h>
 #include <onyx/dev.h>
 #include <onyx/list.h>
 #include <onyx/mm/flush.h>
 #include <onyx/page.h>
-#include <onyx/page_iov.h>
 #include <onyx/types.h>
 
 #include <onyx/slice.hpp>
@@ -27,68 +28,6 @@
 #define BLKDEV_PM_SHUTDOWN 2
 #define BLKDEV_PM_RESET    3
 
-struct blockdev;
-
-using sector_t = uint64_t;
-
-#define BIO_REQ_OP_MASK         (0xff)
-#define BIO_REQ_READ_OP         0
-#define BIO_REQ_WRITE_OP        1
-#define BIO_REQ_DEVICE_SPECIFIC 2
-
-/* BIO flags start at bit 8 since bits 0 - 7 are reserved for operations */
-/* Note that we still have 24 bits for flags, which should be More Than Enough(tm) */
-#define BIO_REQ_DONE     (1 << 8)
-#define BIO_REQ_EIO      (1 << 9)
-#define BIO_REQ_TIMEOUT  (1 << 10)
-#define BIO_REQ_NOT_SUPP (1 << 11)
-
-struct bio_req
-{
-    uint32_t flags;
-    sector_t sector_number;
-    struct page_iov *vec;
-    size_t nr_vecs;
-    size_t curr_vec_index;
-    blockdev *bdev;
-    struct list_head list_node;
-    unsigned long device_specific[4];
-};
-
-/**
- * @brief Allocate a bio_req
- *
- * @param gfp_flags GFP flags
- * @return The allocated, uninitialized bio_req
- */
-struct bio_req *bio_alloc(unsigned int gfp_flags);
-
-/**
- * @brief Free a bio_req
- *
- * @param req Request to free
- */
-void bio_free(struct bio_req *req);
-
-static inline void bio_init(struct bio_req *req)
-{
-    *req = {};
-}
-
-/**
- * @brief Allocate a bio_req and initialize it
- *
- * @param gfp_flags GFP flags
- * @return The allocated, initialized bio_req
- */
-static inline struct bio_req *bio_alloc_and_init(unsigned int gfp_flags)
-{
-    struct bio_req *bio = bio_alloc(gfp_flags);
-    if (likely(bio))
-        bio_init(bio);
-    return bio;
-}
-
 using __blkflush = int (*)(struct blockdev *);
 using __blkpowermanagement = int (*)(int, struct blockdev *);
 
@@ -202,4 +141,25 @@ void partition_setup_disk(struct blockdev *dev);
 
 flush::writeback_dev *bdev_get_wbdev(struct inode *ino);
 
+/**
+ * @brief Handle block IO completion (called from softirqs)
+ *
+ */
+void block_handle_completion();
+
+/**
+ * @brief Queue a pending io_queue to get looked at after the bio_reqs
+ * After completing bio requests, we want to see if we can start up the submission queues again. So
+ * we queue io_queues, and look at them after completing outstanding bio_reqs.
+ * @param queue Queue to complete
+ */
+void block_queue_pending_io_queue(io_queue *queue);
+
+/**
+ * @brief Queue a to-be-completed bio to get completed
+ *
+ * @param bio bio to complete
+ */
+void bio_queue_pending_bio(struct bio_req *bio);
+
 #endif
diff --git a/kernel/include/onyx/block/io-queue.h b/kernel/include/onyx/block/io-queue.h
@@ -9,22 +9,26 @@
 #ifndef _ONYX_BLOCK_IO_QUEUE_H
 #define _ONYX_BLOCK_IO_QUEUE_H
 
+#include <onyx/bio.h>
 #include <onyx/list.h>
 #include <onyx/spinlock.h>
 
 struct bio_req;
 
+#define IO_QUEUE_PENDING_SOFTIRQ (1 << 0)
+
 /**
  * @brief Represents a hardware block IO queue
  *        Serves bio_req's in a non-blocking way.
  *
  */
-class io_queue
+struct io_queue
 {
 protected:
     unsigned int nr_entries_;
-    unsigned int used_entries_;
+    unsigned int used_entries_{0};
     spinlock lock_;
+    unsigned int flags_{0};
     struct list_head req_list_;
 
     /**
@@ -36,7 +40,9 @@ class io_queue
     virtual int device_io_submit(bio_req *req) = 0;
 
 public:
-    constexpr io_queue(unsigned int nr_entries) : nr_entries_{nr_entries}, used_entries_{0}
+    list_head_cpp<io_queue> pending_node_{this};
+
+    constexpr io_queue(unsigned int nr_entries) : nr_entries_{nr_entries}
     {
         INIT_LIST_HEAD(&req_list_);
         spinlock_init(&lock_);
@@ -50,13 +56,50 @@ class io_queue
      */
     bio_req *complete_request(bio_req *req);
 
+    /**
+     * @brief Completes an IO request
+     *
+     * @param req Request
+     */
+    void complete_request2(bio_req *req);
+
     /**
      * @brief Submits a request
      *
      * @param req Request to add to the queue
      * @return 0 on success, negative error codes.
      */
     int submit_request(bio_req *req);
+
+    /**
+     * @brief Set an io_queue as holding pending completed requests.
+     * This queues it in a percpu queue and raises a softirq, if needed.
+     */
+    void set_pending();
+
+    /**
+     * @brief Clear an io_queue's pending flag. The io_queue must be unqueued
+     * from the block_pcpu by then.
+     *
+     */
+    void clear_pending();
+
+    /**
+     * @brief Try to restart the submission queue
+     * Called from softirq context.
+     *
+     */
+    void restart_sq();
+
+    /**
+     * @brief Complete a bio_req
+     * Called from softirq context
+     * @param req Request to complete
+     */
+    virtual void do_complete(bio_req *req)
+    {
+        bio_do_complete(req);
+    }
 };
 
 #endif