text: // SPDX-License-Identifier: GPL-2.0
  /*
   * virtio-fs: Virtio Filesystem
   * Copyright (C) 2018 Red Hat, Inc.
   */
  
  #include <linux/fs.h>
  #include <linux/module.h>
  #include <linux/virtio.h>
  #include <linux/virtio_fs.h>
  #include <linux/delay.h>
  #include <linux/fs_context.h>
  #include <linux/highmem.h>
  #include "fuse_i.h"
  
  /* List of virtio-fs device instances and a lock for the list. Also provides
   * mutual exclusion in device removal and mounting path
   */
  static DEFINE_MUTEX(virtio_fs_mutex);
  static LIST_HEAD(virtio_fs_instances);
  
  enum {
  	VQ_HIPRIO,
  	VQ_REQUEST
  };
  
  /* Per-virtqueue state */
  struct virtio_fs_vq {
  	spinlock_t lock;
  	struct virtqueue *vq;     /* protected by ->lock */
  	struct work_struct done_work;
  	struct list_head queued_reqs;
  	struct list_head end_reqs;	/* End these requests */
  	struct delayed_work dispatch_work;
  	struct fuse_dev *fud;
  	bool connected;
  	long in_flight;
  	struct completion in_flight_zero; /* No inflight requests */
  	char name[24];
  } ____cacheline_aligned_in_smp;
  
  /* A virtio-fs device instance */
  struct virtio_fs {
  	struct kref refcount;
  	struct list_head list;    /* on virtio_fs_instances */
  	char *tag;
  	struct virtio_fs_vq *vqs;
  	unsigned int nvqs;               /* number of virtqueues */
  	unsigned int num_request_queues; /* number of request queues */
  };
  
  struct virtio_fs_forget_req {
  	struct fuse_in_header ih;
  	struct fuse_forget_in arg;
  };
  
  struct virtio_fs_forget {
  	/* This request can be temporarily queued on virt queue */
  	struct list_head list;
  	struct virtio_fs_forget_req req;
  };
  
  struct virtio_fs_req_work {
  	struct fuse_req *req;
  	struct virtio_fs_vq *fsvq;
  	struct work_struct done_work;
  };
  
  static int virtio_fs_enqueue_req(struct virtio_fs_vq *fsvq,
  				 struct fuse_req *req, bool in_flight);
  
  static inline struct virtio_fs_vq *vq_to_fsvq(struct virtqueue *vq)
  {
  	struct virtio_fs *fs = vq->vdev->priv;
  
  	return &fs->vqs[vq->index];
  }
  
  static inline struct fuse_pqueue *vq_to_fpq(struct virtqueue *vq)
  {
  	return &vq_to_fsvq(vq)->fud->pq;
  }
  
  /* Should be called with fsvq->lock held. */
  static inline void inc_in_flight_req(struct virtio_fs_vq *fsvq)
  {
  	fsvq->in_flight++;
  }
  
  /* Should be called with fsvq->lock held. */
  static inline void dec_in_flight_req(struct virtio_fs_vq *fsvq)
  {
  	WARN_ON(fsvq->in_flight <= 0);
  	fsvq->in_flight--;
  	if (!fsvq->in_flight)
  		complete(&fsvq->in_flight_zero);
  }
  
  static void release_virtio_fs_obj(struct kref *ref)
  {
  	struct virtio_fs *vfs = container_of(ref, struct virtio_fs, refcount);
  
  	kfree(vfs->vqs);
  	kfree(vfs);
  }
  
  /* Make sure virtiofs_mutex is held */
  static void virtio_fs_put(struct virtio_fs *fs)
  {
  	kref_put(&fs->refcount, release_virtio_fs_obj);
  }
  
  static void virtio_fs_fiq_release(struct fuse_iqueue *fiq)
  {
  	struct virtio_fs *vfs = fiq->priv;
  
  	mutex_lock(&virtio_fs_mutex);
  	virtio_fs_put(vfs);
  	mutex_unlock(&virtio_fs_mutex);
  }
  
  static void virtio_fs_drain_queue(struct virtio_fs_vq *fsvq)
  {
  	WARN_ON(fsvq->in_flight < 0);
  
  	/* Wait for in flight requests to finish.*/
  	spin_lock(&fsvq->lock);
  	if (fsvq->in_flight) {
  		/* We are holding virtio_fs_mutex. There should not be any
  		 * waiters waiting for completion.
  		 */
  		reinit_completion(&fsvq->in_flight_zero);
  		spin_unlock(&fsvq->lock);
  		wait_for_completion(&fsvq->in_flight_zero);
  	} else {
  		spin_unlock(&fsvq->lock);
  	}
  
  	flush_work(&fsvq->done_work);
  	flush_delayed_work(&fsvq->dispatch_work);
  }
  
  static void virtio_fs_drain_all_queues_locked(struct virtio_fs *fs)
  {
  	struct virtio_fs_vq *fsvq;
  	int i;
  
  	for (i = 0; i < fs->nvqs; i++) {
  		fsvq = &fs->vqs[i];
  		virtio_fs_drain_queue(fsvq);
  	}
  }
  
  static void virtio_fs_drain_all_queues(struct virtio_fs *fs)
  {
  	/* Provides mutual exclusion between ->remove and ->kill_sb
  	 * paths. We don't want both of these draining queue at the
  	 * same time. Current completion logic reinits completion
  	 * and that means there should not be any other thread
  	 * doing reinit or waiting for completion already.
  	 */
  	mutex_lock(&virtio_fs_mutex);
  	virtio_fs_drain_all_queues_locked(fs);
  	mutex_unlock(&virtio_fs_mutex);
  }
  
  static void virtio_fs_start_all_queues(struct virtio_fs *fs)
  {
  	struct virtio_fs_vq *fsvq;
  	int i;
  
  	for (i = 0; i < fs->nvqs; i++) {
  		fsvq = &fs->vqs[i];
  		spin_lock(&fsvq->lock);
  		fsvq->connected = true;
  		spin_unlock(&fsvq->lock);
  	}
  }
  
  /* Add a new instance to the list or return -EEXIST if tag name exists*/
  static int virtio_fs_add_instance(struct virtio_fs *fs)
  {
  	struct virtio_fs *fs2;
  	bool duplicate = false;
  
  	mutex_lock(&virtio_fs_mutex);
  
  	list_for_each_entry(fs2, &virtio_fs_instances, list) {
  		if (strcmp(fs->tag, fs2->tag) == 0)
  			duplicate = true;
  	}
  
  	if (!duplicate)
  		list_add_tail(&fs->list, &virtio_fs_instances);
  
  	mutex_unlock(&virtio_fs_mutex);
  
  	if (duplicate)
  		return -EEXIST;
  	return 0;
  }
  
  /* Return the virtio_fs with a given tag, or NULL */
  static struct virtio_fs *virtio_fs_find_instance(const char *tag)
  {
  	struct virtio_fs *fs;
  
  	mutex_lock(&virtio_fs_mutex);
  
  	list_for_each_entry(fs, &virtio_fs_instances, list) {
  		if (strcmp(fs->tag, tag) == 0) {
  			kref_get(&fs->refcount);
  			goto found;
  		}
  	}
  
  	fs = NULL; /* not found */
  
  found:
  	mutex_unlock(&virtio_fs_mutex);
  
  	return fs;
  }
  
  static void virtio_fs_free_devs(struct virtio_fs *fs)
  {
  	unsigned int i;
  
  	for (i = 0; i < fs->nvqs; i++) {
  		struct virtio_fs_vq *fsvq = &fs->vqs[i];
  
  		if (!fsvq->fud)
  			continue;
  
  		fuse_dev_free(fsvq->fud);
  		fsvq->fud = NULL;
  	}
  }
  
  /* Read filesystem name from virtio config into fs->tag (must kfree()). */
  static int virtio_fs_read_tag(struct virtio_device *vdev, struct virtio_fs *fs)
  {
  	char tag_buf[sizeof_field(struct virtio_fs_config, tag)];
  	char *end;
  	size_t len;
  
  	virtio_cread_bytes(vdev, offsetof(struct virtio_fs_config, tag),
  			   &tag_buf, sizeof(tag_buf));
  	end = memchr(tag_buf, '\0', sizeof(tag_buf));
  	if (end == tag_buf)
  		return -EINVAL; /* empty tag */
  	if (!end)
  		end = &tag_buf[sizeof(tag_buf)];
  
  	len = end - tag_buf;
  	fs->tag = devm_kmalloc(&vdev->dev, len + 1, GFP_KERNEL);
  	if (!fs->tag)
  		return -ENOMEM;
  	memcpy(fs->tag, tag_buf, len);
  	fs->tag[len] = '\0';
  	return 0;
  }
  
  /* Work function for hiprio completion */
  static void virtio_fs_hiprio_done_work(struct work_struct *work)
  {
  	struct virtio_fs_vq *fsvq = container_of(work, struct virtio_fs_vq,
  						 done_work);
  	struct virtqueue *vq = fsvq->vq;
  
  	/* Free completed FUSE_FORGET requests */
  	spin_lock(&fsvq->lock);
  	do {
  		unsigned int len;
  		void *req;
  
  		virtqueue_disable_cb(vq);
  
  		while ((req = virtqueue_get_buf(vq, &len)) != NULL) {
  			kfree(req);
  			dec_in_flight_req(fsvq);
  		}
  	} while (!virtqueue_enable_cb(vq) && likely(!virtqueue_is_broken(vq)));
  	spin_unlock(&fsvq->lock);
  }
  
  static void virtio_fs_request_dispatch_work(struct work_struct *work)
  {
  	struct fuse_req *req;
  	struct virtio_fs_vq *fsvq = container_of(work, struct virtio_fs_vq,
  						 dispatch_work.work);
  	struct fuse_conn *fc = fsvq->fud->fc;
  	int ret;
  
  	pr_debug("virtio-fs: worker %s called.\n", __func__);
  	while (1) {
  		spin_lock(&fsvq->lock);
  		req = list_first_entry_or_null(&fsvq->end_reqs, struct fuse_req,
  					       list);
  		if (!req) {
  			spin_unlock(&fsvq->lock);
  			break;
  		}
  
  		list_del_init(&req->list);
  		spin_unlock(&fsvq->lock);
  		fuse_request_end(fc, req);
  	}
  
  	/* Dispatch pending requests */
  	while (1) {
  		spin_lock(&fsvq->lock);
  		req = list_first_entry_or_null(&fsvq->queued_reqs,
  					       struct fuse_req, list);
  		if (!req) {
  			spin_unlock(&fsvq->lock);
  			return;
  		}
  		list_del_init(&req->list);
  		spin_unlock(&fsvq->lock);
  
  		ret = virtio_fs_enqueue_req(fsvq, req, true);
  		if (ret < 0) {
  			if (ret == -ENOMEM || ret == -ENOSPC) {
  				spin_lock(&fsvq->lock);
  				list_add_tail(&req->list, &fsvq->queued_reqs);
  				schedule_delayed_work(&fsvq->dispatch_work,
  						      msecs_to_jiffies(1));
  				spin_unlock(&fsvq->lock);
  				return;
  			}
  			req->out.h.error = ret;
  			spin_lock(&fsvq->lock);
  			dec_in_flight_req(fsvq);
  			spin_unlock(&fsvq->lock);
  			pr_err("virtio-fs: virtio_fs_enqueue_req() failed %d\n",
  			       ret);
  			fuse_request_end(fc, req);
  		}
  	}
  }
  
  /*
   * Returns 1 if queue is full and sender should wait a bit before sending
   * next request, 0 otherwise.
   */
  static int send_forget_request(struct virtio_fs_vq *fsvq,
  			       struct virtio_fs_forget *forget,
  			       bool in_flight)
  {
  	struct scatterlist sg;
  	struct virtqueue *vq;
  	int ret = 0;
  	bool notify;
  	struct virtio_fs_forget_req *req = &forget->req;
  
  	spin_lock(&fsvq->lock);
  	if (!fsvq->connected) {
  		if (in_flight)
  			dec_in_flight_req(fsvq);
  		kfree(forget);
  		goto out;
  	}
  
  	sg_init_one(&sg, req, sizeof(*req));
  	vq = fsvq->vq;
  	dev_dbg(&vq->vdev->dev, "%s\n", __func__);
  
  	ret = virtqueue_add_outbuf(vq, &sg, 1, forget, GFP_ATOMIC);
  	if (ret < 0) {
  		if (ret == -ENOMEM || ret == -ENOSPC) {
  			pr_debug("virtio-fs: Could not queue FORGET: err=%d. Will try later\n",
  				 ret);
  			list_add_tail(&forget->list, &fsvq->queued_reqs);
  			schedule_delayed_work(&fsvq->dispatch_work,
  					      msecs_to_jiffies(1));
  			if (!in_flight)
  				inc_in_flight_req(fsvq);
  			/* Queue is full */
  			ret = 1;
  		} else {
  			pr_debug("virtio-fs: Could not queue FORGET: err=%d. Dropping it.\n",
  				 ret);
  			kfree(forget);
  			if (in_flight)
  				dec_in_flight_req(fsvq);
  		}
  		goto out;
  	}
  
  	if (!in_flight)
  		inc_in_flight_req(fsvq);
  	notify = virtqueue_kick_prepare(vq);
  	spin_unlock(&fsvq->lock);
  
  	if (notify)
  		virtqueue_notify(vq);
  	return ret;
  out:
  	spin_unlock(&fsvq->lock);
  	return ret;
  }
  
  static void virtio_fs_hiprio_dispatch_work(struct work_struct *work)
  {
  	struct virtio_fs_forget *forget;
  	struct virtio_fs_vq *fsvq = container_of(work, struct virtio_fs_vq,
  						 dispatch_work.work);
  	pr_debug("virtio-fs: worker %s called.\n", __func__);
  	while (1) {
  		spin_lock(&fsvq->lock);
  		forget = list_first_entry_or_null(&fsvq->queued_reqs,
  					struct virtio_fs_forget, list);
  		if (!forget) {
  			spin_unlock(&fsvq->lock);
  			return;
  		}
  
  		list_del(&forget->list);
  		spin_unlock(&fsvq->lock);
  		if (send_forget_request(fsvq, forget, true))
  			return;
  	}
  }
  
  /* Allocate and copy args into req->argbuf */
  static int copy_args_to_argbuf(struct fuse_req *req)
  {
  	struct fuse_args *args = req->args;
  	unsigned int offset = 0;
  	unsigned int num_in;
  	unsigned int num_out;
  	unsigned int len;
  	unsigned int i;
  
  	num_in = args->in_numargs - args->in_pages;
  	num_out = args->out_numargs - args->out_pages;
  	len = fuse_len_args(num_in, (struct fuse_arg *) args->in_args) +
  	      fuse_len_args(num_out, args->out_args);
  
  	req->argbuf = kmalloc(len, GFP_ATOMIC);
  	if (!req->argbuf)
  		return -ENOMEM;
  
  	for (i = 0; i < num_in; i++) {
  		memcpy(req->argbuf + offset,
  		       args->in_args[i].value,
  		       args->in_args[i].size);
  		offset += args->in_args[i].size;
  	}
  
  	return 0;
  }
  
  /* Copy args out of and free req->argbuf */
  static void copy_args_from_argbuf(struct fuse_args *args, struct fuse_req *req)
  {
  	unsigned int remaining;
  	unsigned int offset;
  	unsigned int num_in;
  	unsigned int num_out;
  	unsigned int i;
  
  	remaining = req->out.h.len - sizeof(req->out.h);
  	num_in = args->in_numargs - args->in_pages;
  	num_out = args->out_numargs - args->out_pages;
  	offset = fuse_len_args(num_in, (struct fuse_arg *)args->in_args);
  
  	for (i = 0; i < num_out; i++) {
  		unsigned int argsize = args->out_args[i].size;
  
  		if (args->out_argvar &&
  		    i == args->out_numargs - 1 &&
  		    argsize > remaining) {
  			argsize = remaining;
  		}
  
  		memcpy(args->out_args[i].value, req->argbuf + offset, argsize);
  		offset += argsize;
  
  		if (i != args->out_numargs - 1)
  			remaining -= argsize;
  	}
  
  	/* Store the actual size of the variable-length arg */
  	if (args->out_argvar)
  		args->out_args[args->out_numargs - 1].size = remaining;
  
  	kfree(req->argbuf);
  	req->argbuf = NULL;
  }
  
  /* Work function for request completion */
  static void virtio_fs_request_complete(struct fuse_req *req,
  				       struct virtio_fs_vq *fsvq)
  {
  	struct fuse_pqueue *fpq = &fsvq->fud->pq;
  	struct fuse_conn *fc = fsvq->fud->fc;
  	struct fuse_args *args;
  	struct fuse_args_pages *ap;
  	unsigned int len, i, thislen;
  	struct page *page;
  
  	/*
  	 * TODO verify that server properly follows FUSE protocol
  	 * (oh.uniq, oh.len)
  	 */
  	args = req->args;
  	copy_args_from_argbuf(args, req);
  
  	if (args->out_pages && args->page_zeroing) {
  		len = args->out_args[args->out_numargs - 1].size;
  		ap = container_of(args, typeof(*ap), args);
  		for (i = 0; i < ap->num_pages; i++) {
  			thislen = ap->descs[i].length;
  			if (len < thislen) {
  				WARN_ON(ap->descs[i].offset);
  				page = ap->pages[i];
  				zero_user_segment(page, len, thislen);
  				len = 0;
  			} else {
  				len -= thislen;
  			}
  		}
  	}
  
  	spin_lock(&fpq->lock);
  	clear_bit(FR_SENT, &req->flags);
  	spin_unlock(&fpq->lock);
  
  	fuse_request_end(fc, req);
  	spin_lock(&fsvq->lock);
  	dec_in_flight_req(fsvq);
  	spin_unlock(&fsvq->lock);
  }
  
  static void virtio_fs_complete_req_work(struct work_struct *work)
  {
  	struct virtio_fs_req_work *w =
  		container_of(work, typeof(*w), done_work);
  
  	virtio_fs_request_complete(w->req, w->fsvq);
  	kfree(w);
  }
  
  static void virtio_fs_requests_done_work(struct work_struct *work)
  {
  	struct virtio_fs_vq *fsvq = container_of(work, struct virtio_fs_vq,
  						 done_work);
  	struct fuse_pqueue *fpq = &fsvq->fud->pq;
  	struct virtqueue *vq = fsvq->vq;
  	struct fuse_req *req;
  	struct fuse_req *next;
  	unsigned int len;
  	LIST_HEAD(reqs);
  
  	/* Collect completed requests off the virtqueue */
  	spin_lock(&fsvq->lock);
  	do {
  		virtqueue_disable_cb(vq);
  
  		while ((req = virtqueue_get_buf(vq, &len)) != NULL) {
  			spin_lock(&fpq->lock);
  			list_move_tail(&req->list, &reqs);
  			spin_unlock(&fpq->lock);
  		}
  	} while (!virtqueue_enable_cb(vq) && likely(!virtqueue_is_broken(vq)));
  	spin_unlock(&fsvq->lock);
  
  	/* End requests */
  	list_for_each_entry_safe(req, next, &reqs, list) {
  		list_del_init(&req->list);
  
  		/* blocking async request completes in a worker context */
  		if (req->args->may_block) {
  			struct virtio_fs_req_work *w;
  
  			w = kzalloc(sizeof(*w), GFP_NOFS | __GFP_NOFAIL);
  			INIT_WORK(&w->done_work, virtio_fs_complete_req_work);
  			w->fsvq = fsvq;
  			w->req = req;
  			schedule_work(&w->done_work);
  		} else {
  			virtio_fs_request_complete(req, fsvq);
  		}
  	}
  }
  
  /* Virtqueue interrupt handler */
  static void virtio_fs_vq_done(struct virtqueue *vq)
  {
  	struct virtio_fs_vq *fsvq = vq_to_fsvq(vq);
  
  	dev_dbg(&vq->vdev->dev, "%s %s\n", __func__, fsvq->name);
  
  	schedule_work(&fsvq->done_work);
  }
  
  /* Initialize virtqueues */
  static int virtio_fs_setup_vqs(struct virtio_device *vdev,
  			       struct virtio_fs *fs)
  {
  	struct virtqueue **vqs;
  	vq_callback_t **callbacks;
  	const char **names;
  	unsigned int i;
  	int ret = 0;
  
  	virtio_cread_le(vdev, struct virtio_fs_config, num_request_queues,
  			&fs->num_request_queues);
  	if (fs->num_request_queues == 0)
  		return -EINVAL;
  
  	fs->nvqs = 1 + fs->num_request_queues;
  	fs->vqs = kcalloc(fs->nvqs, sizeof(fs->vqs[VQ_HIPRIO]), GFP_KERNEL);
  	if (!fs->vqs)
  		return -ENOMEM;
  
  	vqs = kmalloc_array(fs->nvqs, sizeof(vqs[VQ_HIPRIO]), GFP_KERNEL);
  	callbacks = kmalloc_array(fs->nvqs, sizeof(callbacks[VQ_HIPRIO]),
  					GFP_KERNEL);
  	names = kmalloc_array(fs->nvqs, sizeof(names[VQ_HIPRIO]), GFP_KERNEL);
  	if (!vqs || !callbacks || !names) {
  		ret = -ENOMEM;
  		goto out;
  	}
  
  	callbacks[VQ_HIPRIO] = virtio_fs_vq_done;
  	snprintf(fs->vqs[VQ_HIPRIO].name, sizeof(fs->vqs[VQ_HIPRIO].name),
  			"hiprio");
  	names[VQ_HIPRIO] = fs->vqs[VQ_HIPRIO].name;
  	INIT_WORK(&fs->vqs[VQ_HIPRIO].done_work, virtio_fs_hiprio_done_work);
  	INIT_LIST_HEAD(&fs->vqs[VQ_HIPRIO].queued_reqs);
  	INIT_LIST_HEAD(&fs->vqs[VQ_HIPRIO].end_reqs);
  	INIT_DELAYED_WORK(&fs->vqs[VQ_HIPRIO].dispatch_work,
  			virtio_fs_hiprio_dispatch_work);
  	init_completion(&fs->vqs[VQ_HIPRIO].in_flight_zero);
  	spin_lock_init(&fs->vqs[VQ_HIPRIO].lock);
  
  	/* Initialize the requests virtqueues */
  	for (i = VQ_REQUEST; i < fs->nvqs; i++) {
  		spin_lock_init(&fs->vqs[i].lock);
  		INIT_WORK(&fs->vqs[i].done_work, virtio_fs_requests_done_work);
  		INIT_DELAYED_WORK(&fs->vqs[i].dispatch_work,
  				  virtio_fs_request_dispatch_work);
  		INIT_LIST_HEAD(&fs->vqs[i].queued_reqs);
  		INIT_LIST_HEAD(&fs->vqs[i].end_reqs);
  		init_completion(&fs->vqs[i].in_flight_zero);
  		snprintf(fs->vqs[i].name, sizeof(fs->vqs[i].name),
  			 "requests.%u", i - VQ_REQUEST);
  		callbacks[i] = virtio_fs_vq_done;
  		names[i] = fs->vqs[i].name;
  	}
  
  	ret = virtio_find_vqs(vdev, fs->nvqs, vqs, callbacks, names, NULL);
  	if (ret < 0)
  		goto out;
  
  	for (i = 0; i < fs->nvqs; i++)
  		fs->vqs[i].vq = vqs[i];
  
  	virtio_fs_start_all_queues(fs);
  out:
  	kfree(names);
  	kfree(callbacks);
  	kfree(vqs);
  	if (ret)
  		kfree(fs->vqs);
  	return ret;
  }
  
  /* Free virtqueues (device must already be reset) */
  static void virtio_fs_cleanup_vqs(struct virtio_device *vdev,
  				  struct virtio_fs *fs)
  {
  	vdev->config->del_vqs(vdev);
  }
  
  static int virtio_fs_probe(struct virtio_device *vdev)
  {
  	struct virtio_fs *fs;
  	int ret;
  
  	fs = kzalloc(sizeof(*fs), GFP_KERNEL);
  	if (!fs)
  		return -ENOMEM;
  	kref_init(&fs->refcount);
  	vdev->priv = fs;
  
  	ret = virtio_fs_read_tag(vdev, fs);
  	if (ret < 0)
  		goto out;
  
  	ret = virtio_fs_setup_vqs(vdev, fs);
  	if (ret < 0)
  		goto out;
  
  	/* TODO vq affinity */
  
  	/* Bring the device online in case the filesystem is mounted and
  	 * requests need to be sent before we return.
  	 */
  	virtio_device_ready(vdev);
  
  	ret = virtio_fs_add_instance(fs);
  	if (ret < 0)
  		goto out_vqs;
  
  	return 0;
  
  out_vqs:
  	vdev->config->reset(vdev);
  	virtio_fs_cleanup_vqs(vdev, fs);
  
  out:
  	vdev->priv = NULL;
  	kfree(fs);
  	return ret;
  }
  
  static void virtio_fs_stop_all_queues(struct virtio_fs *fs)
  {
  	struct virtio_fs_vq *fsvq;
  	int i;
  
  	for (i = 0; i < fs->nvqs; i++) {
  		fsvq = &fs->vqs[i];
  		spin_lock(&fsvq->lock);
  		fsvq->connected = false;
  		spin_unlock(&fsvq->lock);
  	}
  }
  
  static void virtio_fs_remove(struct virtio_device *vdev)
  {
  	struct virtio_fs *fs = vdev->priv;
  
  	mutex_lock(&virtio_fs_mutex);
  	/* This device is going away. No one should get new reference */
  	list_del_init(&fs->list);
  	virtio_fs_stop_all_queues(fs);
  	virtio_fs_drain_all_queues_locked(fs);
  	vdev->config->reset(vdev);
  	virtio_fs_cleanup_vqs(vdev, fs);
  
  	vdev->priv = NULL;
  	/* Put device reference on virtio_fs object */
  	virtio_fs_put(fs);
  	mutex_unlock(&virtio_fs_mutex);
  }
  
  #ifdef CONFIG_PM_SLEEP
  static int virtio_fs_freeze(struct virtio_device *vdev)
  {
  	/* TODO need to save state here */
  	pr_warn("virtio-fs: suspend/resume not yet supported\n");
  	return -EOPNOTSUPP;
  }
  
  static int virtio_fs_restore(struct virtio_device *vdev)
  {
  	 /* TODO need to restore state here */
  	return 0;
  }
  #endif /* CONFIG_PM_SLEEP */
  
  static const struct virtio_device_id id_table[] = {
  	{ VIRTIO_ID_FS, VIRTIO_DEV_ANY_ID },
  	{},
  };
  
  static const unsigned int feature_table[] = {};
  
  static struct virtio_driver virtio_fs_driver = {
  	.driver.name		= KBUILD_MODNAME,
  	.driver.owner		= THIS_MODULE,
  	.id_table		= id_table,
  	.feature_table		= feature_table,
  	.feature_table_size	= ARRAY_SIZE(feature_table),
  	.probe			= virtio_fs_probe,
  	.remove			= virtio_fs_remove,
  #ifdef CONFIG_PM_SLEEP
  	.freeze			= virtio_fs_freeze,
  	.restore		= virtio_fs_restore,
  #endif
  };
  
  static void virtio_fs_wake_forget_and_unlock(struct fuse_iqueue *fiq)
  __releases(fiq->lock)
  {
  	struct fuse_forget_link *link;
  	struct virtio_fs_forget *forget;
  	struct virtio_fs_forget_req *req;
  	struct virtio_fs *fs;
  	struct virtio_fs_vq *fsvq;
  	u64 unique;
  
  	link = fuse_dequeue_forget(fiq, 1, NULL);
  	unique = fuse_get_unique(fiq);
  
  	fs = fiq->priv;
  	fsvq = &fs->vqs[VQ_HIPRIO];
  	spin_unlock(&fiq->lock);
  
  	/* Allocate a buffer for the request */
  	forget = kmalloc(sizeof(*forget), GFP_NOFS | __GFP_NOFAIL);
  	req = &forget->req;
  
  	req->ih = (struct fuse_in_header){
  		.opcode = FUSE_FORGET,
  		.nodeid = link->forget_one.nodeid,
  		.unique = unique,
  		.len = sizeof(*req),
  	};
  	req->arg = (struct fuse_forget_in){
  		.nlookup = link->forget_one.nlookup,
  	};
  
  	send_forget_request(fsvq, forget, false);
  	kfree(link);
  }
  
  static void virtio_fs_wake_interrupt_and_unlock(struct fuse_iqueue *fiq)
  __releases(fiq->lock)
  {
  	/*
  	 * TODO interrupts.
  	 *
  	 * Normal fs operations on a local filesystems aren't interruptible.
  	 * Exceptions are blocking lock operations; for example fcntl(F_SETLKW)
  	 * with shared lock between host and guest.
  	 */
  	spin_unlock(&fiq->lock);
  }
  
  /* Return the number of scatter-gather list elements required */
  static unsigned int sg_count_fuse_req(struct fuse_req *req)
  {
  	struct fuse_args *args = req->args;
  	struct fuse_args_pages *ap = container_of(args, typeof(*ap), args);
  	unsigned int total_sgs = 1 /* fuse_in_header */;
  
  	if (args->in_numargs - args->in_pages)
  		total_sgs += 1;
  
  	if (args->in_pages)
  		total_sgs += ap->num_pages;
  
  	if (!test_bit(FR_ISREPLY, &req->flags))
  		return total_sgs;
  
  	total_sgs += 1 /* fuse_out_header */;
  
  	if (args->out_numargs - args->out_pages)
  		total_sgs += 1;
  
  	if (args->out_pages)
  		total_sgs += ap->num_pages;
  
  	return total_sgs;
  }
  
  /* Add pages to scatter-gather list and return number of elements used */
  static unsigned int sg_init_fuse_pages(struct scatterlist *sg,
  				       struct page **pages,
  				       struct fuse_page_desc *page_descs,
  				       unsigned int num_pages,
  				       unsigned int total_len)
  {
  	unsigned int i;
  	unsigned int this_len;
  
  	for (i = 0; i < num_pages && total_len; i++) {
  		sg_init_table(&sg[i], 1);
  		this_len =  min(page_descs[i].length, total_len);
  		sg_set_page(&sg[i], pages[i], this_len, page_descs[i].offset);
  		total_len -= this_len;
  	}
  
  	return i;
  }
  
  /* Add args to scatter-gather list and return number of elements used */
  static unsigned int sg_init_fuse_args(struct scatterlist *sg,
  				      struct fuse_req *req,
  				      struct fuse_arg *args,
  				      unsigned int numargs,
  				      bool argpages,
  				      void *argbuf,
  				      unsigned int *len_used)
  {
  	struct fuse_args_pages *ap = container_of(req->args, typeof(*ap), args);
  	unsigned int total_sgs = 0;
  	unsigned int len;
  
  	len = fuse_len_args(numargs - argpages, args);
  	if (len)
  		sg_init_one(&sg[total_sgs++], argbuf, len);
  
  	if (argpages)
  		total_sgs += sg_init_fuse_pages(&sg[total_sgs],
  						ap->pages, ap->descs,
  						ap->num_pages,
  						args[numargs - 1].size);
  
  	if (len_used)
  		*len_used = len;
  
  	return total_sgs;
  }
  
  /* Add a request to a virtqueue and kick the device */
  static int virtio_fs_enqueue_req(struct virtio_fs_vq *fsvq,
  				 struct fuse_req *req, bool in_flight)
  {
  	/* requests need at least 4 elements */
  	struct scatterlist *stack_sgs[6];
  	struct scatterlist stack_sg[ARRAY_SIZE(stack_sgs)];
  	struct scatterlist **sgs = stack_sgs;
  	struct scatterlist *sg = stack_sg;
  	struct virtqueue *vq;
  	struct fuse_args *args = req->args;
  	unsigned int argbuf_used = 0;
  	unsigned int out_sgs = 0;
  	unsigned int in_sgs = 0;
  	unsigned int total_sgs;
  	unsigned int i;
  	int ret;
  	bool notify;
  	struct fuse_pqueue *fpq;
  
  	/* Does the sglist fit on the stack? */
  	total_sgs = sg_count_fuse_req(req);
  	if (total_sgs > ARRAY_SIZE(stack_sgs)) {
  		sgs = kmalloc_array(total_sgs, sizeof(sgs[0]), GFP_ATOMIC);
  		sg = kmalloc_array(total_sgs, sizeof(sg[0]), GFP_ATOMIC);
  		if (!sgs || !sg) {
  			ret = -ENOMEM;
  			goto out;
  		}
  	}
  
  	/* Use a bounce buffer since stack args cannot be mapped */
  	ret = copy_args_to_argbuf(req);
  	if (ret < 0)
  		goto out;
  
  	/* Request elements */
  	sg_init_one(&sg[out_sgs++], &req->in.h, sizeof(req->in.h));
  	out_sgs += sg_init_fuse_args(&sg[out_sgs], req,
  				     (struct fuse_arg *)args->in_args,
  				     args->in_numargs, args->in_pages,
  				     req->argbuf, &argbuf_used);
  
  	/* Reply elements */
  	if (test_bit(FR_ISREPLY, &req->flags)) {
  		sg_init_one(&sg[out_sgs + in_sgs++],
  			    &req->out.h, sizeof(req->out.h));
  		in_sgs += sg_init_fuse_args(&sg[out_sgs + in_sgs], req,
  					    args->out_args, args->out_numargs,
  					    args->out_pages,
  					    req->argbuf + argbuf_used, NULL);
  	}
  
  	WARN_ON(out_sgs + in_sgs != total_sgs);
  
  	for (i = 0; i < total_sgs; i++)
  		sgs[i] = &sg[i];
  
  	spin_lock(&fsvq->lock);
  
  	if (!fsvq->connected) {
  		spin_unlock(&fsvq->lock);
  		ret = -ENOTCONN;
  		goto out;
  	}
  
  	vq = fsvq->vq;
  	ret = virtqueue_add_sgs(vq, sgs, out_sgs, in_sgs, req, GFP_ATOMIC);
  	if (ret < 0) {
  		spin_unlock(&fsvq->lock);
  		goto out;
  	}
  
  	/* Request successfully sent. */
  	fpq = &fsvq->fud->pq;
  	spin_lock(&fpq->lock);
  	list_add_tail(&req->list, fpq->processing);
  	spin_unlock(&fpq->lock);
  	set_bit(FR_SENT, &req->flags);
  	/* matches barrier in request_wait_answer() */
  	smp_mb__after_atomic();
  
  	if (!in_flight)
  		inc_in_flight_req(fsvq);
  	notify = virtqueue_kick_prepare(vq);
  
  	spin_unlock(&fsvq->lock);
  
  	if (notify)
  		virtqueue_notify(vq);
  
  out:
  	if (ret < 0 && req->argbuf) {
  		kfree(req->argbuf);
  		req->argbuf = NULL;
  	}
  	if (sgs != stack_sgs) {
  		kfree(sgs);
  		kfree(sg);
  	}
  
  	return ret;
  }
  
  static void virtio_fs_wake_pending_and_unlock(struct fuse_iqueue *fiq)
  __releases(fiq->lock)
  {
  	unsigned int queue_id = VQ_REQUEST; /* TODO multiqueue */
  	struct virtio_fs *fs;
  	struct fuse_req *req;
  	struct virtio_fs_vq *fsvq;
  	int ret;
  
  	WARN_ON(list_empty(&fiq->pending));
  	req = list_last_entry(&fiq->pending, struct fuse_req, list);
  	clear_bit(FR_PENDING, &req->flags);
  	list_del_init(&req->list);
  	WARN_ON(!list_empty(&fiq->pending));
  	spin_unlock(&fiq->lock);
  
  	fs = fiq->priv;
  
  	pr_debug("%s: opcode %u unique %#llx nodeid %#llx in.len %u out.len %u\n",
  		  __func__, req->in.h.opcode, req->in.h.unique,
  		 req->in.h.nodeid, req->in.h.len,
  		 fuse_len_args(req->args->out_numargs, req->args->out_args));
  
  	fsvq = &fs->vqs[queue_id];
  	ret = virtio_fs_enqueue_req(fsvq, req, false);
  	if (ret < 0) {
  		if (ret == -ENOMEM || ret == -ENOSPC) {
  			/*
  			 * Virtqueue full. Retry submission from worker
  			 * context as we might be holding fc->bg_lock.
  			 */
  			spin_lock(&fsvq->lock);
  			list_add_tail(&req->list, &fsvq->queued_reqs);
  			inc_in_flight_req(fsvq);
  			schedule_delayed_work(&fsvq->dispatch_work,
  						msecs_to_jiffies(1));
  			spin_unlock(&fsvq->lock);
  			return;
  		}
  		req->out.h.error = ret;
  		pr_err("virtio-fs: virtio_fs_enqueue_req() failed %d\n", ret);
  
  		/* Can't end request in submission context. Use a worker */
  		spin_lock(&fsvq->lock);
  		list_add_tail(&req->list, &fsvq->end_reqs);
  		schedule_delayed_work(&fsvq->dispatch_work, 0);
  		spin_unlock(&fsvq->lock);
  		return;
  	}
  }
  
  static const struct fuse_iqueue_ops virtio_fs_fiq_ops = {
  	.wake_forget_and_unlock		= virtio_fs_wake_forget_and_unlock,
  	.wake_interrupt_and_unlock	= virtio_fs_wake_interrupt_and_unlock,
  	.wake_pending_and_unlock	= virtio_fs_wake_pending_and_unlock,
  	.release			= virtio_fs_fiq_release,
  };
  
  static int virtio_fs_fill_super(struct super_block *sb)
  {
  	struct fuse_conn *fc = get_fuse_conn_super(sb);
  	struct virtio_fs *fs = fc->iq.priv;
  	unsigned int i;
  	int err;
  	struct fuse_fs_context ctx = {
  		.rootmode = S_IFDIR,
  		.default_permissions = 1,
  		.allow_other = 1,
  		.max_read = UINT_MAX,
  		.blksize = 512,
  		.destroy = true,
  		.no_control = true,
  		.no_force_umount = true,
  		.no_mount_options = true,
  	};
  
  	mutex_lock(&virtio_fs_mutex);
  
  	/* After holding mutex, make sure virtiofs device is still there.
  	 * Though we are holding a reference to it, drive ->remove might
  	 * still have cleaned up virtual queues. In that case bail out.
  	 */
  	err = -EINVAL;
  	if (list_empty(&fs->list)) {
  		pr_info("virtio-fs: tag <%s> not found\n", fs->tag);
  		goto err;
  	}
  
  	err = -ENOMEM;
  	/* Allocate fuse_dev for hiprio and notification queues */
  	for (i = 0; i < fs->nvqs; i++) {
  		struct virtio_fs_vq *fsvq = &fs->vqs[i];
  
  		fsvq->fud = fuse_dev_alloc();
  		if (!fsvq->fud)
  			goto err_free_fuse_devs;
  	}
  
  	/* virtiofs allocates and installs its own fuse devices */
  	ctx.fudptr = NULL;
  	err = fuse_fill_super_common(sb, &ctx);
  	if (err < 0)
  		goto err_free_fuse_devs;
  
  	for (i = 0; i < fs->nvqs; i++) {
  		struct virtio_fs_vq *fsvq = &fs->vqs[i];
  
  		fuse_dev_install(fsvq->fud, fc);
  	}
  
  	/* Previous unmount will stop all queues. Start these again */
  	virtio_fs_start_all_queues(fs);
  	fuse_send_init(fc);
  	mutex_unlock(&virtio_fs_mutex);
  	return 0;
  
  err_free_fuse_devs:
  	virtio_fs_free_devs(fs);
  err:
  	mutex_unlock(&virtio_fs_mutex);
  	return err;
  }
  
  static void virtio_kill_sb(struct super_block *sb)
  {
  	struct fuse_conn *fc = get_fuse_conn_super(sb);
  	struct virtio_fs *vfs;
  	struct virtio_fs_vq *fsvq;
  
  	/* If mount failed, we can still be called without any fc */
  	if (!fc)
  		return fuse_kill_sb_anon(sb);
  
  	vfs = fc->iq.priv;
  	fsvq = &vfs->vqs[VQ_HIPRIO];
  
  	/* Stop forget queue. Soon destroy will be sent */
  	spin_lock(&fsvq->lock);
  	fsvq->connected = false;
  	spin_unlock(&fsvq->lock);
  	virtio_fs_drain_all_queues(vfs);
  
  	fuse_kill_sb_anon(sb);
  
  	/* fuse_kill_sb_anon() must have sent destroy. Stop all queues
  	 * and drain one more time and free fuse devices. Freeing fuse
  	 * devices will drop their reference on fuse_conn and that in
  	 * turn will drop its reference on virtio_fs object.
  	 */
  	virtio_fs_stop_all_queues(vfs);
  	virtio_fs_drain_all_queues(vfs);
  	virtio_fs_free_devs(vfs);
  }
  
  static int virtio_fs_test_super(struct super_block *sb,
  				struct fs_context *fsc)
  {
  	struct fuse_conn *fc = fsc->s_fs_info;
  
  	return fc->iq.priv == get_fuse_conn_super(sb)->iq.priv;
  }
  
  static int virtio_fs_set_super(struct super_block *sb,
  			       struct fs_context *fsc)
  {
  	int err;
  
  	err = get_anon_bdev(&sb->s_dev);
  	if (!err)
  		fuse_conn_get(fsc->s_fs_info);
  
  	return err;
  }
  
  static int virtio_fs_get_tree(struct fs_context *fsc)
  {
  	struct virtio_fs *fs;
  	struct super_block *sb;
  	struct fuse_conn *fc;
  	int err;
  
  	/* This gets a reference on virtio_fs object. This ptr gets installed
  	 * in fc->iq->priv. Once fuse_conn is going away, it calls ->put()
  	 * to drop the reference to this object.
  	 */
  	fs = virtio_fs_find_instance(fsc->source);
  	if (!fs) {
  		pr_info("virtio-fs: tag <%s> not found\n", fsc->source);
  		return -EINVAL;
  	}
  
  	fc = kzalloc(sizeof(struct fuse_conn), GFP_KERNEL);
  	if (!fc) {
  		mutex_lock(&virtio_fs_mutex);
  		virtio_fs_put(fs);
  		mutex_unlock(&virtio_fs_mutex);
  		return -ENOMEM;
  	}
  
  	fuse_conn_init(fc, get_user_ns(current_user_ns()), &virtio_fs_fiq_ops,
  		       fs);
  	fc->release = fuse_free_conn;
  	fc->delete_stale = true;
  
  	fsc->s_fs_info = fc;
  	sb = sget_fc(fsc, virtio_fs_test_super, virtio_fs_set_super);
  	fuse_conn_put(fc);
  	if (IS_ERR(sb))
  		return PTR_ERR(sb);
  
  	if (!sb->s_root) {
  		err = virtio_fs_fill_super(sb);
  		if (err) {
  			deactivate_locked_super(sb);
  			return err;
  		}
  
  		sb->s_flags |= SB_ACTIVE;
  	}
  
  	WARN_ON(fsc->root);
  	fsc->root = dget(sb->s_root);
  	return 0;
  }
  
  static const struct fs_context_operations virtio_fs_context_ops = {
  	.get_tree	= virtio_fs_get_tree,
  };
  
  static int virtio_fs_init_fs_context(struct fs_context *fsc)
  {
  	fsc->ops = &virtio_fs_context_ops;
  	return 0;
  }
  
  static struct file_system_type virtio_fs_type = {
  	.owner		= THIS_MODULE,
  	.name		= "virtiofs",
  	.init_fs_context = virtio_fs_init_fs_context,
  	.kill_sb	= virtio_kill_sb,
  };
  
  static int __init virtio_fs_init(void)
  {
  	int ret;
  
  	ret = register_virtio_driver(&virtio_fs_driver);
  	if (ret < 0)
  		return ret;
  
  	ret = register_filesystem(&virtio_fs_type);
  	if (ret < 0) {
  		unregister_virtio_driver(&virtio_fs_driver);
  		return ret;
  	}
  
  	return 0;
  }
  module_init(virtio_fs_init);
  
  static void __exit virtio_fs_exit(void)
  {
  	unregister_filesystem(&virtio_fs_type);
  	unregister_virtio_driver(&virtio_fs_driver);
  }
  module_exit(virtio_fs_exit);
  
  MODULE_AUTHOR("Stefan Hajnoczi <stefanha@redhat.com>");
  MODULE_DESCRIPTION("Virtio Filesystem");
  MODULE_LICENSE("GPL");
  MODULE_ALIAS_FS(KBUILD_MODNAME);
  MODULE_DEVICE_TABLE(virtio, id_table);
  