aboutsummaryrefslogtreecommitdiff
path: root/hw/block/dataplane
diff options
context:
space:
mode:
Diffstat (limited to 'hw/block/dataplane')
-rw-r--r--hw/block/dataplane/Makefile.objs2
-rw-r--r--hw/block/dataplane/ioq.c117
-rw-r--r--hw/block/dataplane/ioq.h57
-rw-r--r--hw/block/dataplane/virtio-blk.c256
4 files changed, 101 insertions, 331 deletions
diff --git a/hw/block/dataplane/Makefile.objs b/hw/block/dataplane/Makefile.objs
index 9da2eb82ba..e786f66421 100644
--- a/hw/block/dataplane/Makefile.objs
+++ b/hw/block/dataplane/Makefile.objs
@@ -1 +1 @@
-obj-y += ioq.o virtio-blk.o
+obj-y += virtio-blk.o
diff --git a/hw/block/dataplane/ioq.c b/hw/block/dataplane/ioq.c
deleted file mode 100644
index f709f87ed6..0000000000
--- a/hw/block/dataplane/ioq.c
+++ /dev/null
@@ -1,117 +0,0 @@
-/*
- * Linux AIO request queue
- *
- * Copyright 2012 IBM, Corp.
- * Copyright 2012 Red Hat, Inc. and/or its affiliates
- *
- * Authors:
- * Stefan Hajnoczi <stefanha@redhat.com>
- *
- * This work is licensed under the terms of the GNU GPL, version 2 or later.
- * See the COPYING file in the top-level directory.
- *
- */
-
-#include "ioq.h"
-
-void ioq_init(IOQueue *ioq, int fd, unsigned int max_reqs)
-{
- int rc;
-
- ioq->fd = fd;
- ioq->max_reqs = max_reqs;
-
- memset(&ioq->io_ctx, 0, sizeof ioq->io_ctx);
- rc = io_setup(max_reqs, &ioq->io_ctx);
- if (rc != 0) {
- fprintf(stderr, "ioq io_setup failed %d\n", rc);
- exit(1);
- }
-
- rc = event_notifier_init(&ioq->io_notifier, 0);
- if (rc != 0) {
- fprintf(stderr, "ioq io event notifier creation failed %d\n", rc);
- exit(1);
- }
-
- ioq->freelist = g_malloc0(sizeof ioq->freelist[0] * max_reqs);
- ioq->freelist_idx = 0;
-
- ioq->queue = g_malloc0(sizeof ioq->queue[0] * max_reqs);
- ioq->queue_idx = 0;
-}
-
-void ioq_cleanup(IOQueue *ioq)
-{
- g_free(ioq->freelist);
- g_free(ioq->queue);
-
- event_notifier_cleanup(&ioq->io_notifier);
- io_destroy(ioq->io_ctx);
-}
-
-EventNotifier *ioq_get_notifier(IOQueue *ioq)
-{
- return &ioq->io_notifier;
-}
-
-struct iocb *ioq_get_iocb(IOQueue *ioq)
-{
- /* Underflow cannot happen since ioq is sized for max_reqs */
- assert(ioq->freelist_idx != 0);
-
- struct iocb *iocb = ioq->freelist[--ioq->freelist_idx];
- ioq->queue[ioq->queue_idx++] = iocb;
- return iocb;
-}
-
-void ioq_put_iocb(IOQueue *ioq, struct iocb *iocb)
-{
- /* Overflow cannot happen since ioq is sized for max_reqs */
- assert(ioq->freelist_idx != ioq->max_reqs);
-
- ioq->freelist[ioq->freelist_idx++] = iocb;
-}
-
-struct iocb *ioq_rdwr(IOQueue *ioq, bool read, struct iovec *iov,
- unsigned int count, long long offset)
-{
- struct iocb *iocb = ioq_get_iocb(ioq);
-
- if (read) {
- io_prep_preadv(iocb, ioq->fd, iov, count, offset);
- } else {
- io_prep_pwritev(iocb, ioq->fd, iov, count, offset);
- }
- io_set_eventfd(iocb, event_notifier_get_fd(&ioq->io_notifier));
- return iocb;
-}
-
-int ioq_submit(IOQueue *ioq)
-{
- int rc = io_submit(ioq->io_ctx, ioq->queue_idx, ioq->queue);
- ioq->queue_idx = 0; /* reset */
- return rc;
-}
-
-int ioq_run_completion(IOQueue *ioq, IOQueueCompletion *completion,
- void *opaque)
-{
- struct io_event events[ioq->max_reqs];
- int nevents, i;
-
- do {
- nevents = io_getevents(ioq->io_ctx, 0, ioq->max_reqs, events, NULL);
- } while (nevents < 0 && errno == EINTR);
- if (nevents < 0) {
- return nevents;
- }
-
- for (i = 0; i < nevents; i++) {
- ssize_t ret = ((uint64_t)events[i].res2 << 32) | events[i].res;
-
- completion(events[i].obj, ret, opaque);
- ioq_put_iocb(ioq, events[i].obj);
- }
- return nevents;
-}
diff --git a/hw/block/dataplane/ioq.h b/hw/block/dataplane/ioq.h
deleted file mode 100644
index b49b5de7f4..0000000000
--- a/hw/block/dataplane/ioq.h
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Linux AIO request queue
- *
- * Copyright 2012 IBM, Corp.
- * Copyright 2012 Red Hat, Inc. and/or its affiliates
- *
- * Authors:
- * Stefan Hajnoczi <stefanha@redhat.com>
- *
- * This work is licensed under the terms of the GNU GPL, version 2 or later.
- * See the COPYING file in the top-level directory.
- *
- */
-
-#ifndef IOQ_H
-#define IOQ_H
-
-#include <libaio.h>
-#include "qemu/event_notifier.h"
-
-typedef struct {
- int fd; /* file descriptor */
- unsigned int max_reqs; /* max length of freelist and queue */
-
- io_context_t io_ctx; /* Linux AIO context */
- EventNotifier io_notifier; /* Linux AIO eventfd */
-
- /* Requests can complete in any order so a free list is necessary to manage
- * available iocbs.
- */
- struct iocb **freelist; /* free iocbs */
- unsigned int freelist_idx;
-
- /* Multiple requests are queued up before submitting them all in one go */
- struct iocb **queue; /* queued iocbs */
- unsigned int queue_idx;
-} IOQueue;
-
-void ioq_init(IOQueue *ioq, int fd, unsigned int max_reqs);
-void ioq_cleanup(IOQueue *ioq);
-EventNotifier *ioq_get_notifier(IOQueue *ioq);
-struct iocb *ioq_get_iocb(IOQueue *ioq);
-void ioq_put_iocb(IOQueue *ioq, struct iocb *iocb);
-struct iocb *ioq_rdwr(IOQueue *ioq, bool read, struct iovec *iov,
- unsigned int count, long long offset);
-int ioq_submit(IOQueue *ioq);
-
-static inline unsigned int ioq_num_queued(IOQueue *ioq)
-{
- return ioq->queue_idx;
-}
-
-typedef void IOQueueCompletion(struct iocb *iocb, ssize_t ret, void *opaque);
-int ioq_run_completion(IOQueue *ioq, IOQueueCompletion *completion,
- void *opaque);
-
-#endif /* IOQ_H */
diff --git a/hw/block/dataplane/virtio-blk.c b/hw/block/dataplane/virtio-blk.c
index e49c2536b1..c10b7b70fb 100644
--- a/hw/block/dataplane/virtio-blk.c
+++ b/hw/block/dataplane/virtio-blk.c
@@ -17,7 +17,6 @@
#include "qemu/thread.h"
#include "qemu/error-report.h"
#include "hw/virtio/dataplane/vring.h"
-#include "ioq.h"
#include "block/block.h"
#include "hw/virtio/virtio-blk.h"
#include "virtio-blk.h"
@@ -25,20 +24,14 @@
#include "hw/virtio/virtio-bus.h"
#include "qom/object_interfaces.h"
-enum {
- SEG_MAX = 126, /* maximum number of I/O segments */
- VRING_MAX = SEG_MAX + 2, /* maximum number of vring descriptors */
- REQ_MAX = VRING_MAX, /* maximum number of requests in the vring,
- * is VRING_MAX / 2 with traditional and
- * VRING_MAX with indirect descriptors */
-};
-
typedef struct {
- struct iocb iocb; /* Linux AIO control block */
+ VirtIOBlockDataPlane *s;
QEMUIOVector *inhdr; /* iovecs for virtio_blk_inhdr */
VirtQueueElement *elem; /* saved data from the virtqueue */
- struct iovec *bounce_iov; /* used if guest buffers are unaligned */
- QEMUIOVector *read_qiov; /* for read completion /w bounce buffer */
+ QEMUIOVector qiov; /* original request iovecs */
+ struct iovec bounce_iov; /* used if guest buffers are unaligned */
+ QEMUIOVector bounce_qiov; /* bounce buffer iovecs */
+ bool read; /* read or write? */
} VirtIOBlockRequest;
struct VirtIOBlockDataPlane {
@@ -47,7 +40,6 @@ struct VirtIOBlockDataPlane {
bool stopping;
VirtIOBlkConf *blk;
- int fd; /* image file descriptor */
VirtIODevice *vdev;
Vring vring; /* virtqueue vring */
@@ -61,16 +53,8 @@ struct VirtIOBlockDataPlane {
IOThread *iothread;
IOThread internal_iothread_obj;
AioContext *ctx;
- EventNotifier io_notifier; /* Linux AIO completion */
EventNotifier host_notifier; /* doorbell */
- IOQueue ioqueue; /* Linux AIO queue (should really be per
- IOThread) */
- VirtIOBlockRequest requests[REQ_MAX]; /* pool of requests, managed by the
- queue */
-
- unsigned int num_reqs;
-
/* Operation blocker on BDS */
Error *blocker;
};
@@ -85,33 +69,28 @@ static void notify_guest(VirtIOBlockDataPlane *s)
event_notifier_set(s->guest_notifier);
}
-static void complete_request(struct iocb *iocb, ssize_t ret, void *opaque)
+static void complete_rdwr(void *opaque, int ret)
{
- VirtIOBlockDataPlane *s = opaque;
- VirtIOBlockRequest *req = container_of(iocb, VirtIOBlockRequest, iocb);
+ VirtIOBlockRequest *req = opaque;
struct virtio_blk_inhdr hdr;
int len;
- if (likely(ret >= 0)) {
+ if (likely(ret == 0)) {
hdr.status = VIRTIO_BLK_S_OK;
- len = ret;
+ len = req->qiov.size;
} else {
hdr.status = VIRTIO_BLK_S_IOERR;
len = 0;
}
- trace_virtio_blk_data_plane_complete_request(s, req->elem->index, ret);
+ trace_virtio_blk_data_plane_complete_request(req->s, req->elem->index, ret);
- if (req->read_qiov) {
- assert(req->bounce_iov);
- qemu_iovec_from_buf(req->read_qiov, 0, req->bounce_iov->iov_base, len);
- qemu_iovec_destroy(req->read_qiov);
- g_slice_free(QEMUIOVector, req->read_qiov);
+ if (req->read && req->bounce_iov.iov_base) {
+ qemu_iovec_from_buf(&req->qiov, 0, req->bounce_iov.iov_base, len);
}
- if (req->bounce_iov) {
- qemu_vfree(req->bounce_iov->iov_base);
- g_slice_free(struct iovec, req->bounce_iov);
+ if (req->bounce_iov.iov_base) {
+ qemu_vfree(req->bounce_iov.iov_base);
}
qemu_iovec_from_buf(req->inhdr, 0, &hdr, sizeof(hdr));
@@ -122,9 +101,9 @@ static void complete_request(struct iocb *iocb, ssize_t ret, void *opaque)
* written to, but for virtio-blk it seems to be the number of bytes
* transferred plus the status bytes.
*/
- vring_push(&s->vring, req->elem, len + sizeof(hdr));
- req->elem = NULL;
- s->num_reqs--;
+ vring_push(&req->s->vring, req->elem, len + sizeof(hdr));
+ notify_guest(req->s);
+ g_slice_free(VirtIOBlockRequest, req);
}
static void complete_request_early(VirtIOBlockDataPlane *s, VirtQueueElement *elem,
@@ -155,51 +134,87 @@ static void do_get_id_cmd(VirtIOBlockDataPlane *s,
complete_request_early(s, elem, inhdr, VIRTIO_BLK_S_OK);
}
-static int do_rdwr_cmd(VirtIOBlockDataPlane *s, bool read,
- struct iovec *iov, unsigned iov_cnt,
- long long offset, VirtQueueElement *elem,
- QEMUIOVector *inhdr)
+static void do_rdwr_cmd(VirtIOBlockDataPlane *s, bool read,
+ struct iovec *iov, unsigned iov_cnt,
+ int64_t sector_num, VirtQueueElement *elem,
+ QEMUIOVector *inhdr)
{
- struct iocb *iocb;
- QEMUIOVector qiov;
- struct iovec *bounce_iov = NULL;
- QEMUIOVector *read_qiov = NULL;
-
- qemu_iovec_init_external(&qiov, iov, iov_cnt);
- if (!bdrv_qiov_is_aligned(s->blk->conf.bs, &qiov)) {
- void *bounce_buffer = qemu_blockalign(s->blk->conf.bs, qiov.size);
-
- if (read) {
- /* Need to copy back from bounce buffer on completion */
- read_qiov = g_slice_new(QEMUIOVector);
- qemu_iovec_init(read_qiov, iov_cnt);
- qemu_iovec_concat_iov(read_qiov, iov, iov_cnt, 0, qiov.size);
- } else {
- qemu_iovec_to_buf(&qiov, 0, bounce_buffer, qiov.size);
+ VirtIOBlockRequest *req = g_slice_new0(VirtIOBlockRequest);
+ QEMUIOVector *qiov;
+ int nb_sectors;
+
+ /* Fill in virtio block metadata needed for completion */
+ req->s = s;
+ req->elem = elem;
+ req->inhdr = inhdr;
+ req->read = read;
+ qemu_iovec_init_external(&req->qiov, iov, iov_cnt);
+
+ qiov = &req->qiov;
+
+ if (!bdrv_qiov_is_aligned(s->blk->conf.bs, qiov)) {
+ void *bounce_buffer = qemu_blockalign(s->blk->conf.bs, qiov->size);
+
+ /* Populate bounce buffer with data for writes */
+ if (!read) {
+ qemu_iovec_to_buf(qiov, 0, bounce_buffer, qiov->size);
}
/* Redirect I/O to aligned bounce buffer */
- bounce_iov = g_slice_new(struct iovec);
- bounce_iov->iov_base = bounce_buffer;
- bounce_iov->iov_len = qiov.size;
- iov = bounce_iov;
- iov_cnt = 1;
+ req->bounce_iov.iov_base = bounce_buffer;
+ req->bounce_iov.iov_len = qiov->size;
+ qemu_iovec_init_external(&req->bounce_qiov, &req->bounce_iov, 1);
+ qiov = &req->bounce_qiov;
}
- iocb = ioq_rdwr(&s->ioqueue, read, iov, iov_cnt, offset);
+ nb_sectors = qiov->size / BDRV_SECTOR_SIZE;
- /* Fill in virtio block metadata needed for completion */
- VirtIOBlockRequest *req = container_of(iocb, VirtIOBlockRequest, iocb);
+ if (read) {
+ bdrv_aio_readv(s->blk->conf.bs, sector_num, qiov, nb_sectors,
+ complete_rdwr, req);
+ } else {
+ bdrv_aio_writev(s->blk->conf.bs, sector_num, qiov, nb_sectors,
+ complete_rdwr, req);
+ }
+}
+
+static void complete_flush(void *opaque, int ret)
+{
+ VirtIOBlockRequest *req = opaque;
+ unsigned char status;
+
+ if (ret == 0) {
+ status = VIRTIO_BLK_S_OK;
+ } else {
+ status = VIRTIO_BLK_S_IOERR;
+ }
+
+ complete_request_early(req->s, req->elem, req->inhdr, status);
+ g_slice_free(VirtIOBlockRequest, req);
+}
+
+static void do_flush_cmd(VirtIOBlockDataPlane *s, VirtQueueElement *elem,
+ QEMUIOVector *inhdr)
+{
+ VirtIOBlockRequest *req = g_slice_new(VirtIOBlockRequest);
+ req->s = s;
req->elem = elem;
req->inhdr = inhdr;
- req->bounce_iov = bounce_iov;
- req->read_qiov = read_qiov;
- return 0;
+
+ bdrv_aio_flush(s->blk->conf.bs, complete_flush, req);
+}
+
+static void do_scsi_cmd(VirtIOBlockDataPlane *s, VirtQueueElement *elem,
+ QEMUIOVector *inhdr)
+{
+ int status;
+
+ status = virtio_blk_handle_scsi_req(VIRTIO_BLK(s->vdev), elem);
+ complete_request_early(s, elem, inhdr, status);
}
-static int process_request(IOQueue *ioq, VirtQueueElement *elem)
+static int process_request(VirtIOBlockDataPlane *s, VirtQueueElement *elem)
{
- VirtIOBlockDataPlane *s = container_of(ioq, VirtIOBlockDataPlane, ioqueue);
struct iovec *iov = elem->out_sg;
struct iovec *in_iov = elem->in_sg;
unsigned out_num = elem->out_num;
@@ -234,25 +249,23 @@ static int process_request(IOQueue *ioq, VirtQueueElement *elem)
switch (outhdr.type) {
case VIRTIO_BLK_T_IN:
- do_rdwr_cmd(s, true, in_iov, in_num, outhdr.sector * 512, elem, inhdr);
+ do_rdwr_cmd(s, true, in_iov, in_num,
+ outhdr.sector * 512 / BDRV_SECTOR_SIZE,
+ elem, inhdr);
return 0;
case VIRTIO_BLK_T_OUT:
- do_rdwr_cmd(s, false, iov, out_num, outhdr.sector * 512, elem, inhdr);
+ do_rdwr_cmd(s, false, iov, out_num,
+ outhdr.sector * 512 / BDRV_SECTOR_SIZE,
+ elem, inhdr);
return 0;
case VIRTIO_BLK_T_SCSI_CMD:
- /* TODO support SCSI commands */
- complete_request_early(s, elem, inhdr, VIRTIO_BLK_S_UNSUPP);
+ do_scsi_cmd(s, elem, inhdr);
return 0;
case VIRTIO_BLK_T_FLUSH:
- /* TODO fdsync not supported by Linux AIO, do it synchronously here! */
- if (qemu_fdatasync(s->fd) < 0) {
- complete_request_early(s, elem, inhdr, VIRTIO_BLK_S_IOERR);
- } else {
- complete_request_early(s, elem, inhdr, VIRTIO_BLK_S_OK);
- }
+ do_flush_cmd(s, elem, inhdr);
return 0;
case VIRTIO_BLK_T_GET_ID:
@@ -274,7 +287,6 @@ static void handle_notify(EventNotifier *e)
VirtQueueElement *elem;
int ret;
- unsigned int num_queued;
event_notifier_test_and_clear(&s->host_notifier);
for (;;) {
@@ -291,7 +303,7 @@ static void handle_notify(EventNotifier *e)
trace_virtio_blk_data_plane_process_request(s, elem->out_num,
elem->in_num, elem->index);
- if (process_request(&s->ioqueue, elem) < 0) {
+ if (process_request(s, elem) < 0) {
vring_set_broken(&s->vring);
vring_free_element(elem);
ret = -EFAULT;
@@ -306,44 +318,10 @@ static void handle_notify(EventNotifier *e)
if (vring_enable_notification(s->vdev, &s->vring)) {
break;
}
- } else { /* ret == -ENOBUFS or fatal error, iovecs[] is depleted */
- /* Since there are no iovecs[] left, stop processing for now. Do
- * not re-enable guest->host notifies since the I/O completion
- * handler knows to check for more vring descriptors anyway.
- */
+ } else { /* fatal error */
break;
}
}
-
- num_queued = ioq_num_queued(&s->ioqueue);
- if (num_queued > 0) {
- s->num_reqs += num_queued;
-
- int rc = ioq_submit(&s->ioqueue);
- if (unlikely(rc < 0)) {
- fprintf(stderr, "ioq_submit failed %d\n", rc);
- exit(1);
- }
- }
-}
-
-static void handle_io(EventNotifier *e)
-{
- VirtIOBlockDataPlane *s = container_of(e, VirtIOBlockDataPlane,
- io_notifier);
-
- event_notifier_test_and_clear(&s->io_notifier);
- if (ioq_run_completion(&s->ioqueue, complete_request, s) > 0) {
- notify_guest(s);
- }
-
- /* If there were more requests than iovecs, the vring will not be empty yet
- * so check again. There should now be enough resources to process more
- * requests.
- */
- if (unlikely(vring_more_avail(&s->vring))) {
- handle_notify(&s->host_notifier);
- }
}
/* Context: QEMU global mutex held */
@@ -352,7 +330,6 @@ void virtio_blk_data_plane_create(VirtIODevice *vdev, VirtIOBlkConf *blk,
Error **errp)
{
VirtIOBlockDataPlane *s;
- int fd;
Error *local_err = NULL;
*dataplane = NULL;
@@ -361,18 +338,6 @@ void virtio_blk_data_plane_create(VirtIODevice *vdev, VirtIOBlkConf *blk,
return;
}
- if (blk->scsi) {
- error_setg(errp,
- "device is incompatible with x-data-plane, use scsi=off");
- return;
- }
-
- if (blk->config_wce) {
- error_setg(errp, "device is incompatible with x-data-plane, "
- "use config-wce=off");
- return;
- }
-
/* If dataplane is (re-)enabled while the guest is running there could be
* block jobs that can conflict.
*/
@@ -383,16 +348,8 @@ void virtio_blk_data_plane_create(VirtIODevice *vdev, VirtIOBlkConf *blk,
return;
}
- fd = raw_get_aio_fd(blk->conf.bs);
- if (fd < 0) {
- error_setg(errp, "drive is incompatible with x-data-plane, "
- "use format=raw,cache=none,aio=native");
- return;
- }
-
s = g_new0(VirtIOBlockDataPlane, 1);
s->vdev = vdev;
- s->fd = fd;
s->blk = blk;
if (blk->iothread) {
@@ -437,7 +394,6 @@ void virtio_blk_data_plane_start(VirtIOBlockDataPlane *s)
BusState *qbus = BUS(qdev_get_parent_bus(DEVICE(s->vdev)));
VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
VirtQueue *vq;
- int i;
if (s->started) {
return;
@@ -470,24 +426,18 @@ void virtio_blk_data_plane_start(VirtIOBlockDataPlane *s)
}
s->host_notifier = *virtio_queue_get_host_notifier(vq);
- /* Set up ioqueue */
- ioq_init(&s->ioqueue, s->fd, REQ_MAX);
- for (i = 0; i < ARRAY_SIZE(s->requests); i++) {
- ioq_put_iocb(&s->ioqueue, &s->requests[i].iocb);
- }
- s->io_notifier = *ioq_get_notifier(&s->ioqueue);
-
s->starting = false;
s->started = true;
trace_virtio_blk_data_plane_start(s);
+ bdrv_set_aio_context(s->blk->conf.bs, s->ctx);
+
/* Kick right away to begin processing requests already in vring */
event_notifier_set(virtio_queue_get_host_notifier(vq));
/* Get this show started by hooking up our callbacks */
aio_context_acquire(s->ctx);
aio_set_event_notifier(s->ctx, &s->host_notifier, handle_notify);
- aio_set_event_notifier(s->ctx, &s->io_notifier, handle_io);
aio_context_release(s->ctx);
}
@@ -507,13 +457,8 @@ void virtio_blk_data_plane_stop(VirtIOBlockDataPlane *s)
/* Stop notifications for new requests from guest */
aio_set_event_notifier(s->ctx, &s->host_notifier, NULL);
- /* Complete pending requests */
- while (s->num_reqs > 0) {
- aio_poll(s->ctx, true);
- }
-
- /* Stop ioq callbacks (there are no pending requests left) */
- aio_set_event_notifier(s->ctx, &s->io_notifier, NULL);
+ /* Drain and switch bs back to the QEMU main loop */
+ bdrv_set_aio_context(s->blk->conf.bs, qemu_get_aio_context());
aio_context_release(s->ctx);
@@ -522,7 +467,6 @@ void virtio_blk_data_plane_stop(VirtIOBlockDataPlane *s)
*/
vring_teardown(&s->vring, s->vdev, 0);
- ioq_cleanup(&s->ioqueue);
k->set_host_notifier(qbus->parent, 0, false);
/* Clean up guest notifier (irq) */
generated by cgit v1.2.3 (git 2.26.2) at 2024-11-27 09:22:08 +0000