/*-
 *   BSD LICENSE
 *
 *   Copyright (c) Intel Corporation. All rights reserved.
 *   Copyright (c) 2017, Western Digital Corporation or its affiliates.
 *
 *   Redistribution and use in source and binary forms, with or without
 *   modification, are permitted provided that the following conditions
 *   are met:
 *
 *     * Redistributions of source code must retain the above copyright
 *       notice, this list of conditions and the following disclaimer.
 *     * Redistributions in binary form must reproduce the above copyright
 *       notice, this list of conditions and the following disclaimer in
 *       the documentation and/or other materials provided with the
 *       distribution.
 *     * Neither the name of Intel Corporation nor the names of its
 *       contributors may be used to endorse or promote products derived
 *       from this software without specific prior written permission.
 *
 *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

#include "nvme_internal.h"

/*
 * Allocate a request descriptor from the queue pair free list.
 */
static struct nvme_request *nvme_alloc_request(struct nvme_qpair *qpair)
{
	struct nvme_request *req;

	pthread_mutex_lock(&qpair->lock);

	req = STAILQ_FIRST(&qpair->free_req);
	if (req) {
		STAILQ_REMOVE_HEAD(&qpair->free_req, stailq);
		memset(&req->cmd, 0, sizeof(struct nvme_cmd));
	}

	pthread_mutex_unlock(&qpair->lock);

	return req;
}

static void nvme_request_cb_complete_child(void *child_arg,
					   const struct nvme_cpl *cpl)
{
	struct nvme_request *child = child_arg;
	struct nvme_request *parent = child->parent;

	nvme_request_remove_child(parent, child);

	if (nvme_cpl_is_error(cpl))
		memcpy(&parent->parent_status, cpl, sizeof(*cpl));

	if (parent->child_reqs == 0) {
		if (parent->cb_fn)
			parent->cb_fn(parent->cb_arg, &parent->parent_status);
		nvme_request_free_locked(parent);
	}
}

void nvme_request_completion_poll_cb(void *arg, const struct nvme_cpl *cpl)
{
	struct nvme_completion_poll_status *status = arg;

	memcpy(&status->cpl, cpl, sizeof(*cpl));
	status->done = true;
}

int nvme_request_pool_construct(struct nvme_qpair *qpair)
{
	struct nvme_request *req;
	unsigned int i;

	qpair->num_reqs = qpair->trackers * NVME_IO_ENTRIES_VS_TRACKERS_RATIO;
	qpair->reqs = calloc(qpair->num_reqs, sizeof(struct nvme_request));
	if (!qpair->reqs) {
		nvme_err("QPair %d: allocate %u requests failed\n",
			 (int)qpair->id, qpair->num_reqs);
		return ENOMEM;
	}

	nvme_info("QPair %d: %d requests in pool\n",
		  (int)qpair->id,
		  (int)qpair->num_reqs);

	for(i = 0; i < qpair->num_reqs; i++) {
		req = &qpair->reqs[i];
		req->qpair = qpair;
		STAILQ_INSERT_TAIL(&qpair->free_req, req, stailq);
		req++;
	}

	return 0;
}

void nvme_request_pool_destroy(struct nvme_qpair *qpair)
{
	struct nvme_request *req;
	unsigned int n = 0;

	while ((req = STAILQ_FIRST(&qpair->free_req))) {
		STAILQ_REMOVE_HEAD(&qpair->free_req, stailq);
		n++;
	}

	if (n != qpair->num_reqs)
		nvme_err("QPair %d: Freed %d/%d requests\n",
			 (int)qpair->id, n, (int)qpair->num_reqs);

	free(qpair->reqs);
}

struct nvme_request *nvme_request_allocate(struct nvme_qpair *qpair,
					   const struct nvme_payload *payload,
					   uint32_t payload_size,
					   nvme_cmd_cb cb_fn,
					   void *cb_arg)
{
	struct nvme_request *req;

	req = nvme_alloc_request(qpair);
	if (req == NULL)
		return NULL;

	/*
	 * Only memset up to (but not including) the children TAILQ_ENTRY.
	 * Children, and following members, are only used as part of I/O
	 * splitting so we avoid memsetting them until it is actually needed.
	 * They will be initialized in nvme_request_add_child()
	 * if the request is split.
	 */
	memset(req, 0, offsetof(struct nvme_request, children));
	req->cb_fn = cb_fn;
	req->cb_arg = cb_arg;
	req->payload = *payload;
	req->payload_size = payload_size;

	return req;
}

struct nvme_request *nvme_request_allocate_contig(struct nvme_qpair *qpair,
						  void *buffer,
						  uint32_t payload_size,
						  nvme_cmd_cb cb_fn,
						  void *cb_arg)
{
	struct nvme_payload payload;

	payload.type = NVME_PAYLOAD_TYPE_CONTIG;
	payload.u.contig = buffer;
	payload.md = NULL;

	return nvme_request_allocate(qpair, &payload, payload_size,
				     cb_fn, cb_arg);
}

struct nvme_request *nvme_request_allocate_null(struct nvme_qpair *qpair,
						nvme_cmd_cb cb_fn, void *cb_arg)
{
	return nvme_request_allocate_contig(qpair, NULL, 0, cb_fn, cb_arg);
}

void nvme_request_free_locked(struct nvme_request *req)
{
	nvme_assert(req->child_reqs == 0, "Number of child request not 0\n");

	STAILQ_INSERT_HEAD(&req->qpair->free_req, req, stailq);
}

void nvme_request_free(struct nvme_request *req)
{
	pthread_mutex_lock(&req->qpair->lock);

	nvme_request_free_locked(req);

	pthread_mutex_unlock(&req->qpair->lock);
}

void nvme_request_add_child(struct nvme_request *parent,
			    struct nvme_request *child)
{
	if (parent->child_reqs == 0) {
		/*
		 * Defer initialization of the children TAILQ since it falls
		 * on a separate cacheline.  This ensures we do not touch this
		 * cacheline except on request splitting cases, which are
		 * relatively rare.
		 */
		TAILQ_INIT(&parent->children);
		parent->parent = NULL;
		memset(&parent->parent_status, 0, sizeof(struct nvme_cpl));
	}

	parent->child_reqs++;
	TAILQ_INSERT_TAIL(&parent->children, child, child_tailq);
	child->parent = parent;
	child->cb_fn = nvme_request_cb_complete_child;
	child->cb_arg = child;
}

void nvme_request_remove_child(struct nvme_request *parent,
			       struct nvme_request *child)
{
	nvme_assert(child->parent == parent, "child->parent != parent\n");
	nvme_assert(parent->child_reqs != 0, "child_reqs is 0\n");

	parent->child_reqs--;
	TAILQ_REMOVE(&parent->children, child, child_tailq);
}