Prep for DriverKit

docs/api/support/mem.rst

@@ -3,4 +3,4 @@
 Memory Allocation Helpers
 =========================
 
-.. kernel-doc:: include/vfn/support/mem.h
+.. kernel-doc:: include/vfn/support/platform/posix/mem.h

include/vfn/iommu/dmabuf.h

@@ -0,0 +1,84 @@
+/* SPDX-License-Identifier: LGPL-2.1-or-later or MIT */
+
+/*
+ * This file is part of libvfn.
+ *
+ * Copyright (C) 2023 The libvfn Authors. All Rights Reserved.
+ *
+ * This library (libvfn) is dual licensed under the GNU Lesser General
+ * Public License version 2.1 or later or the MIT license. See the
+ * COPYING and LICENSE files for more information.
+ */
+
+#ifndef LIBVFN_IOMMU_DMABUF_H
+#define LIBVFN_IOMMU_DMABUF_H
+
+/**
+ * DOC: DMA-buffer helpers
+ *
+ * This provides a helper for allocating and mapping DMA buffers.
+ *
+ * &struct iommu_dmabuf is also registered as an "autovar" and can be
+ * automatically unmapped and deallocated when going out of scope. For this
+ * reason, this header is not included in <vfn/iommu.h> and must explicitly be
+ * included.
+ *
+ * Required includes:
+ *   #include <vfn/iommu.h>
+ *   #include <vfn/support/autoptr.h>
+ *   #include <vfn/iommu/dmabuf.h>
+ */
+
+
+/**
+ * struct iommu_dmabuf - DMA buffer abstraction
+ * @ctx: &struct iommu_ctx
+ * @vaddr: data buffer
+ * @iova: mapped address
+ * @len: length of @vaddr
+ *
+ * Convenience wrapper around a mapped data buffer.
+ */
+struct iommu_dmabuf {
+	struct iommu_ctx *ctx;
+
+	void *vaddr;
+	uint64_t iova;
+	ssize_t len;
+};
+
+/**
+ * iommu_get_dmabuf - Allocate and map a DMA buffer
+ * @ctx: &struct iommu_ctx
+ * @buffer: uninitialized &struct iommu_dmabuf
+ * @len: desired minimum length
+ * @flags: combination of enum iommu_map_flags
+ *
+ * Allocate at least @len bytes and map the buffer within the IOVA address space
+ * described by @ctx. The actual allocated and mapped length may be larger than
+ * requestes due to alignment requirements.
+ *
+ * Return: On success, returns ``0``; on error, returns ``-1`` and sets
+ * ``errno``.
+ */
+int iommu_get_dmabuf(struct iommu_ctx *ctx, struct iommu_dmabuf *buffer, size_t len,
+		     unsigned long flags);
+
+/**
+ * iommu_put_dmabuf - Unmap and deallocate a DMA buffer
+ * @buffer: &struct iommu_dmabuf
+ *
+ * Unmap the buffer and deallocate it.
+ */
+void iommu_put_dmabuf(struct iommu_dmabuf *buffer);
+
+static inline void __do_iommu_put_dmabuf(void *p)
+{
+	struct iommu_dmabuf *buffer = (struct iommu_dmabuf *)p;
+
+	iommu_put_dmabuf(buffer);
+}
+
+DEFINE_AUTOVAR_STRUCT(iommu_dmabuf, __do_iommu_put_dmabuf);
+
+#endif /* LIBVFN_IOMMU_DMABUF_H */

include/vfn/nvme.h

@@ -36,6 +36,7 @@ extern "C" {
 #include <vfn/trace.h>
 #include <vfn/trace/events.h>
 #include <vfn/iommu.h>
+#include <vfn/iommu/dmabuf.h>
 #include <vfn/vfio.h>
 #include <vfn/nvme/types.h>
 #include <vfn/nvme/queue.h>

include/vfn/nvme/ctrl.h

@@ -82,8 +82,8 @@ struct nvme_ctrl {
 	 * @dbbuf: doorbell buffers
 	 */
 	struct {
-		void *doorbells;
-		void *eventidxs;
+		struct iommu_dmabuf doorbells;
+		struct iommu_dmabuf eventidxs;
 	} dbbuf;
 
 	/**

include/vfn/nvme/queue.h

@@ -27,8 +27,7 @@ struct nvme_dbbuf {
  */
 struct nvme_cq {
 	/* private: */
-	void *vaddr;
-	uint64_t iova;
+	struct iommu_dmabuf mem;
 
 	int id;
 	uint16_t head;
@@ -51,13 +50,8 @@ struct nvme_sq {
 	/* private: */
 	struct nvme_cq *cq;
 
-	void *vaddr;
-	uint64_t iova;
-
-	struct {
-		void *vaddr;
-		uint64_t iova;
-	} pages;
+	struct iommu_dmabuf mem;
+	struct iommu_dmabuf pages;
 
 	uint16_t tail, ptail;
 	int qsize;
@@ -84,7 +78,7 @@ struct nvme_sq {
  */
 static inline void nvme_sq_post(struct nvme_sq *sq, const union nvme_cmd *sqe)
 {
-	memcpy(sq->vaddr + (sq->tail << NVME_SQES), sqe, 1 << NVME_SQES);
+	memcpy(sq->mem.vaddr + (sq->tail << NVME_SQES), sqe, 1 << NVME_SQES);
 
 	trace_guard(NVME_SQ_POST) {
 		trace_emit("sqid %d tail %d\n", sq->id, sq->tail);
@@ -174,7 +168,7 @@ static inline void nvme_sq_exec(struct nvme_sq *sq, const union nvme_cmd *sqe)
  */
 static inline struct nvme_cqe *nvme_cq_head(struct nvme_cq *cq)
 {
-	return (struct nvme_cqe *)(cq->vaddr + (cq->head << NVME_CQES));
+	return (struct nvme_cqe *)(cq->mem.vaddr + (cq->head << NVME_CQES));
 }
 
 /**

include/vfn/nvme/util.h

@@ -55,7 +55,12 @@ static inline bool nvme_cqe_ok(struct nvme_cqe *cqe)
  *
  * Return: ``0`` when no error in @cqe, otherwise ``-1`` and set ``errno``.
  */
-int nvme_set_errno_from_cqe(struct nvme_cqe *cqe);
+static inline int nvme_set_errno_from_cqe(struct nvme_cqe *cqe)
+{
+	errno = le16_to_cpu(cqe->sfp) >> 1 ? EIO : 0;
+
+	return errno ? -1 : 0;
+}
 
 /**
  * nvme_aer - Submit an Asynchronous Event Request command

include/vfn/support/autoptr.h

@@ -14,11 +14,11 @@
 #define LIBVFN_SUPPORT_AUTOPTR_H
 
 /**
- * DOC: glib-style auto pointer
+ * DOC: glib-style automatic cleanup.
  *
- * The __autoptr() provides a general way of "cleaning up" when going out of
- * scope. Inspired by glib, but simplified a lot (at the expence of
- * flexibility).
+ * The __autoptr() and __autovar_s() provides a general way of "cleaning up"
+ * when going out of scope. Inspired by glib, but simplified a lot (at the
+ * expence of flexibility).
  */
 
 #define __AUTOPTR_CLEANUP(t) __autoptr_cleanup_##t
@@ -27,7 +27,7 @@
 /**
  * DEFINE_AUTOPTR - Defines the appropriate cleanup function for a pointer type
  * @t: type name
- * @cleanup: function to be called to cleanup type
+ * @cleanup: function to be called to clean up type
  *
  * Defines a function ``__autoptr_cleanup_##t`` that will call @cleanup when
  * invoked.
@@ -51,4 +51,34 @@
  */
 #define __autoptr(t) __attribute__((cleanup(__AUTOPTR_CLEANUP(t)))) __AUTOPTR_T(t)
 
+#define __AUTOVAR_STRUCT_CLEANUP(t) __autovar_struct_cleanup_##t
+#define __AUTOVAR_STRUCT_T(t) __autovar_struct_##t
+
+/**
+ * DEFINE_AUTOVAR_STRUCT - Defines the appropriate cleanup function for a struct
+ *                         type
+ * @t: type name
+ * @cleanup: function to be called to clean up type
+ *
+ * Defines a function ``__autovar_struct_cleanup_##t`` that will call @cleanup
+ * when invoked.
+ */
+#define DEFINE_AUTOVAR_STRUCT(t, cleanup) \
+	typedef struct t __AUTOVAR_STRUCT_T(t); \
+	\
+	static inline void __AUTOVAR_STRUCT_CLEANUP(t) (struct t *p) \
+	{ \
+		(cleanup)(p); \
+	}
+
+/**
+ * __autovar_s - Helper to declare a struct variable with automatic cleanup
+ * @t: type name
+ *
+ * Declares a struct-type variable that is cleaned up when the variable goes out
+ * of scope.  How to clean up the type must have been previously declared using
+ * DEFINE_AUTOPTR().
+ */
+#define __autovar_s(t) __attribute__((cleanup(__AUTOVAR_STRUCT_CLEANUP(t)))) __AUTOVAR_STRUCT_T(t)
+
 #endif /* LIBVFN_SUPPORT_AUTOPTR_H */

include/vfn/support/mem.h

@@ -40,41 +40,11 @@ static inline size_t __abort_on_overflow(unsigned int n, size_t sz)
 	return n * sz;
 }
 
-/**
- * xmalloc - version of malloc that cannot fail
- * @sz: number of bytes to allocate
- *
- * Call malloc, but only return NULL when @sz is zero. Otherwise, abort.
- *
- * Return: pointer to allocated memory
- */
-static inline void *xmalloc(size_t sz)
-{
-	void *mem;
-
-	if (unlikely(!sz))
-		return NULL;
-
-	mem = malloc(sz);
-	if (unlikely(!mem))
-		backtrace_abort();
-
-	return mem;
-}
-
-static inline void *zmalloc(size_t sz)
-{
-	void *mem;
-
-	if (unlikely(!sz))
-		return NULL;
-
-	mem = calloc(1, sz);
-	if (unlikely(!mem))
-		backtrace_abort();
-
-	return mem;
-}
+#ifdef __APPLE__
+# include "vfn/support/platform/driverkit/mem.h"
+#else
+# include "vfn/support/platform/posix/mem.h"
+#endif
 
 static inline void *mallocn(unsigned int n, size_t sz)
 {
@@ -98,17 +68,6 @@ static inline void *zmallocn(unsigned int n, size_t sz)
 	return zmalloc(n * sz);
 }
 
-static inline void *reallocn(void *mem, unsigned int n, size_t sz)
-{
-	if (would_overflow(n, sz)) {
-		fprintf(stderr, "allocation of %d * %zu bytes would overflow\n", n, sz);
-
-		backtrace_abort();
-	}
-
-	return realloc(mem, n * sz);
-}
-
 #define _new_t(t, n, f) \
 	((t *) f(n, sizeof(t)))
 

include/vfn/support/platform/driverkit/mem.h

@@ -0,0 +1,65 @@
+/* SPDX-License-Identifier: LGPL-2.1-or-later or MIT */
+
+/*
+ * This file is part of libvfn.
+ *
+ * Copyright (C) 2024 The libvfn Authors. All Rights Reserved.
+ *
+ * This library (libvfn) is dual licensed under the GNU Lesser General
+ * Public License version 2.1 or later or the MIT license. See the
+ * COPYING and LICENSE files for more information.
+ */
+
+#ifndef LIBVFN_SUPPORT_PLATFORM_DRIVERKIT_MEM_H
+#define LIBVFN_SUPPORT_PLATFORM_DRIVERKIT_MEM_H
+
+#include <DriverKit/IOLib.h>
+#include <DriverKit/IOBufferMemoryDescriptor.h>
+
+static inline void *xmalloc(size_t sz)
+{
+	void *mem;
+
+	if (unlikely(!sz))
+		return NULL;
+
+	sz += sizeof(uint64_t);
+
+	mem = IOMalloc(sz);
+	if (unlikely(!mem))
+		backtrace_abort();
+
+	/* prepend length of allocationl (we need it on free) */
+	*(uint64_t *)mem = sz;
+
+	return (void *)(((char *)mem) + sizeof(uint64_t));
+}
+
+static inline void *zmalloc(size_t sz)
+{
+	void *mem;
+
+	if (unlikely(!sz))
+		return NULL;
+
+	sz += sizeof(uint64_t);
+
+	mem = IOMallocZero(sz);
+	if (unlikely(!mem))
+		backtrace_abort();
+
+	/* prepend length of allocationl (we need it on free) */
+	*(uint64_t *)mem = sz;
+
+	return (void *)(((char *)mem) + sizeof(uint64_t));
+}
+
+static inline void free(void *ptr)
+{
+	void *real_ptr = (void *)(((uint8_t *)ptr) - sizeof(uint64_t));
+	uint64_t len = *(uint64_t *)real_ptr;
+
+	IOFree(real_ptr, len);
+}
+
+#endif /* LIBVFN_SUPPORT_PLATFORM_DRIVERKIT_MEM_H */