/***************************************************************************
 *
 *   SPDX-License-Identifier: BSD-3-Clause
 *   Copyright(c) 2007-2026 Intel Corporation
 * 
 *   These contents may have been developed with support from one or more
 *   Intel-operated generative artificial intelligence solutions.
 *
 ***************************************************************************/
#include <string.h>
#include <errno.h>
#include <unistd.h>
#include "adf_user_ring.h"
#include "adf_user_cfg.h"
#include <qae_mem.h>
#include <adf_platform_common.h>
#include <adf_platform_acceldev_common.h>
#include <adf_platform_acceldev_gen4.h>
#include <icp_platform.h>
#include <icp_platform_user.h>
#include "adf_io_ring.h"

static void *mem_alloc(Cpa32U size,
                       Cpa32U node,
                       Cpa32U alignment,
                       adf_dev_ring_handle_t *ring)
{
    return qaeMemAllocNUMA(size, node, alignment);
}

static void mem_free(void **ptr, adf_dev_ring_handle_t *ring)
{
    qaeMemFreeNUMA(ptr);
}

static uint32_t validateRingSize(uint32_t num_msgs_on_ring,
                                 uint32_t msg_size_in_bytes,
                                 uint32_t *modulo_value)
{
    const uint32_t size_in_bytes = num_msgs_on_ring * msg_size_in_bytes;
    const uint32_t max_ring_bytes = ICP_ET_SIZE_TO_BYTES(ICP_RINGSIZE_MEG_4);
    uint32_t size_config = 0;
    uint32_t kbytes = 0;
    uint32_t k_size = 0;

    if (0 >= num_msgs_on_ring || 0 >= msg_size_in_bytes ||
        0 != modulo(msg_size_in_bytes, 4))
    {
        ADF_ERROR("Invalid Input: Num messages on ring=%d, "
                  "Msg size(bytes)=%d. Using 16K\n",
                  num_msgs_on_ring,
                  msg_size_in_bytes);
        /* use default ring size */
        size_config = ICP_RINGSIZE_KILO_16;
        *modulo_value = MODULO_SHIFT_FOR_16K;
    }
    else if (max_ring_bytes >= size_in_bytes)
    {
        /*  under range will auto give minimum size
            or valid range calculate the highest bit set
            first right shift to smallest ring size */
        kbytes = ((size_in_bytes - 1) >> MODULO_SHIFT_FOR_1K);
        while (kbytes > 0)
        {
            kbytes = kbytes >> 1;
            k_size++;
        }
        size_config = ICP_RINGSIZE_KILO_1 + k_size;
        *modulo_value = MODULO_SHIFT_FOR_1K + k_size;
    }
    else
    {
        /* larger than supported maximum */
        /* set for maximum size */
        size_config = ICP_RINGSIZE_MEG_4;
        *modulo_value = MODULO_SHIFT_FOR_4M;
    }

    return size_config;
}

static int adf_reserve_ring(adf_dev_bank_handle_t *bank, uint32_t ring_number)
{
    int status = -1;

    if (NULL == bank->user_bank_lock)
    {
        ADF_ERROR("user_bank_lock is not initialized\n");
        return status;
    }

    status = ICP_MUTEX_LOCK(&bank->user_bank_lock);
    if (status)
    {
        ADF_ERROR("Failed to lock bank with error %d\n", status);
        return status;
    }

    if (!(bank->ring_mask & (1 << ring_number)))
    {
        bank->ring_mask |= (1 << ring_number);
    }
    else
        status = -EBUSY;

    ICP_MUTEX_UNLOCK(&bank->user_bank_lock);

    return status;
}

static void adf_unreserve_ring(adf_dev_bank_handle_t *bank,
                               uint32_t ring_number)
{
    int status;

    status = ICP_MUTEX_LOCK(&bank->user_bank_lock);
    if (status)
    {
        ADF_ERROR("Failed to lock bank with error %d\n", status);
        return;
    }
    bank->ring_mask &= ~(1 << ring_number);
    ICP_MUTEX_UNLOCK(&bank->user_bank_lock);
}

CpaStatus adf_user_put_msg(adf_dev_ring_handle_t *ring,
                           uint32_t *inBuf,
                           uint64_t *seq_num)
{
    CpaStatus status = CPA_STATUS_SUCCESS;
    uint32_t *targetAddr;
    int64_t flight;
    ICP_CHECK_FOR_NULL_PARAM(ring);
    ICP_CHECK_FOR_NULL_PARAM(inBuf);
    ICP_CHECK_FOR_NULL_PARAM(ring->accel_dev);

#ifndef ICP_WITHOUT_QP_SUBMISSION_LOCK
    status = ICP_MUTEX_LOCK(&ring->user_lock);
    if (status)
    {
        ADF_ERROR("Failed to lock bank with error %d\n", status);
        return CPA_STATUS_FAIL;
    }
#endif

    /* Check if there is enough space in the ring */
    flight = __sync_add_and_fetch(ring->in_flight, 1);
    if (flight > ring->max_requests_inflight)
    {
        __sync_sub_and_fetch(ring->in_flight, 1);
        status = CPA_STATUS_RETRY;
        goto adf_user_put_msg_exit;
    }

    targetAddr = (uint32_t *)(((UARCH_INT)ring->ring_virt_addr) + ring->tail);
    if (ring->message_size == ADF_MSG_SIZE_64_BYTES)
    {
        adf_memcpy64(targetAddr, inBuf);
    }
    else if (ring->message_size == ADF_MSG_SIZE_128_BYTES)
    {
        adf_memcpy128(targetAddr, inBuf);
    }
    else
    {
        status = CPA_STATUS_FAIL;
        goto adf_user_put_msg_exit;
    }

    if (IS_RING_IN_WQ_MODE(ring))
    {
        /* Update shadow copy values */
        ring->tail = modulo((ring->tail + ring->message_size), ring->modulo);
        /* and the config space of the device */
        WRITE_CSR_RING_TAIL(
            ring->csr_addr, ring->bank_offset, ring->ring_num, ring->tail);
    }

    ring->csrTailOffset = ring->tail;

    if (NULL != seq_num)
        *seq_num = ring->send_seq;

    ring->send_seq++;

adf_user_put_msg_exit:
#ifndef ICP_WITHOUT_QP_SUBMISSION_LOCK
    ICP_MUTEX_UNLOCK(&ring->user_lock);
#endif
    return status;
}

int32_t adf_user_check_ring_error(adf_dev_ring_handle_t *ring)
{
    uint8_t *csr_base_addr = NULL;
    uint32_t ring_stat = 0;
    device_type_t deviceType;

    ICP_CHECK_FOR_NULL_PARAM(ring);

    deviceType = ring->accel_dev->deviceType;

    /* if generation is not supporting ring error reporting assume there was no
     * error */
    if (IS_QAT_GEN2(deviceType))
        return 0;

    csr_base_addr = ((uint8_t *)ring->csr_addr);

    ring_stat = READ_CSR_RING_STATUS_GEN4(csr_base_addr, ring->bank_offset);

    if (ring_stat & CSR_RING_STAT_RL_EXCEPTION_MASK)
        return -EINTR;

    if (ring_stat & CSR_RING_STAT_RL_HALT_MASK)
        return -EL2HLT;

    return 0;
}

/*
 * Check function used for response rings. It will check the response rings
 * until the number of in-flight requests to determine whether there is
 * responses remained on the response ring.
 */
CpaBoolean adf_user_check_resp_ring(adf_dev_ring_handle_t *ring)
{
    int32_t num_checked_msg = 0;
    int32_t cur_head = ring->head;
    volatile uint32_t *msg = NULL;

    while (num_checked_msg < *ring->in_flight)
    {
        msg = (uint32_t *)(((UARCH_INT)ring->ring_virt_addr) + cur_head);

        if (EMPTY_RING_SIG_WORD != *msg)
        {
            return CPA_FALSE;
        }
        cur_head = modulo((cur_head + ring->message_size), ring->modulo);
        num_checked_msg++;
    }

    return CPA_TRUE;
}

/*
 * Notify function used for polling. Messages are read until the ring is
 * empty or the response quota has been fulfilled.
 * If the response quota is zero, messages are read until the ring is drained.
 */
CpaStatus adf_user_notify_msgs_poll(adf_dev_ring_handle_t *ring)
{
    volatile uint32_t *msg = NULL;
    uint32_t msg_counter = 0, response_quota;

    response_quota = (ring->ringResponseQuota != 0) ? ring->ringResponseQuota
                                                    : ICP_NO_RESPONSE_QUOTA;
    /* point to where the next message should be */
    msg = (uint32_t *)(((UARCH_INT)ring->ring_virt_addr) + ring->head);

    /* If there are valid messages then process them */
    while ((*msg != EMPTY_RING_SIG_WORD) && (msg_counter < response_quota))
    {
        /* Invoke the callback for the message */
        ring->callback((uint32_t *)msg);

        /* Mark the message as processed */
        *msg = EMPTY_RING_SIG_WORD;

        /* Advance the head offset and handle wraparound */
        ring->head = modulo((ring->head + ring->message_size), ring->modulo);
        msg_counter++;
        /* Point to where the next message should be */
        msg = (uint32_t *)(((UARCH_INT)ring->ring_virt_addr) + ring->head);
    }

    /* Update the head CSR if any messages were processed */
    if (msg_counter > 0)
    {
        /* May need to do this earlier to prevent perf impact in multi-threaded
         * scenarios */
        __sync_sub_and_fetch(ring->in_flight, msg_counter);

        /* Coalesce head writes to reduce impact of MMIO write, except if
         * interrupt method is enabled cause otherwise it would keep triggering
         * new interrupts over and over again */
        if (msg_counter > ring->coal_write_count ||
            ICP_RESP_TYPE_IRQ == ring->resp)
        {
            ring->coal_write_count = ring->min_resps_per_head_write;
            if (IS_RING_IN_WQ_MODE(ring))
            {
                WRITE_CSR_RING_HEAD(ring->csr_addr,
                                    ring->bank_offset,
                                    ring->ring_num,
                                    ring->head);
            }
        }
        else
        {
            /* Not enough responses have been processed to warrant the cost
             * of a head write. Updating the count for the next time. */
            ring->coal_write_count -= msg_counter;
        }
    }
    else
    {
        return CPA_STATUS_RETRY;
    }

    return CPA_STATUS_SUCCESS;
}

static void adf_populate_ring_config(adf_dev_ring_handle_t *ring,
                                     adf_dev_bank_handle_t *bank,
                                     uint32_t ring_size_cfg)
{
    uint64_t ring_base_cfg, ring_config;
    uint8_t nearly_full_wm = ICP_RING_NEAR_WATERMARK_512;
    uint8_t nearly_empty_wm = ICP_RING_NEAR_WATERMARK_0;
    uint32_t *csr_base_addr = ring->csr_addr;
    device_type_t deviceType;

    deviceType = ring->accel_dev->deviceType;

    if (bank->tx_rings_mask & (1 << ring->ring_num))
    {
        ring_config = BUILD_RING_CONFIG(ring_size_cfg);
    }
    else
    {
        ring_config = BUILD_RESP_RING_CONFIG(
            ring_size_cfg, nearly_full_wm, nearly_empty_wm);
    }

    if (!IS_QAT_GEN2(deviceType))
    {
        ring_base_cfg =
            BUILD_RING_BASE_ADDR_GEN4(ring->ring_phys_base_addr, ring_size_cfg);
        WRITE_CSR_RING_BASE_GEN4(
            csr_base_addr, ring->bank_offset, ring->ring_num, ring_base_cfg);
        WRITE_CSR_RING_CONFIG_GEN4(
            csr_base_addr, ring->bank_offset, ring->ring_num, ring_config);
    }
    else
    {
        ring_base_cfg =
            BUILD_RING_BASE_ADDR(ring->ring_phys_base_addr, ring_size_cfg);
        WRITE_CSR_RING_BASE(ring->bank_offset, ring->ring_num, ring_base_cfg);
        WRITE_CSR_RING_CONFIG(ring->bank_offset, ring->ring_num, ring_config);
    }
}

static int32_t adf_init_ring_internal(adf_dev_ring_handle_t *ring,
                                      adf_dev_bank_handle_t *bank,
                                      uint32_t ring_num,
                                      uint32_t *csr_base_addr,
                                      uint32_t num_msgs,
                                      uint32_t msg_size,
                                      int nodeid)
{

    uint32_t modulo = 0;
    uint32_t ring_size_cfg = validateRingSize(num_msgs, msg_size, &modulo);
    uint32_t ring_size_bytes = ICP_ET_SIZE_TO_BYTES(ring_size_cfg);
    uint32_t max_space = ring_size_bytes;

    ICP_CHECK_FOR_NULL_PARAM(ring->accel_dev);

    /* Exclusive access to one ring */
    if (adf_reserve_ring(bank, ring_num))
    {
        ADF_ERROR(
            "Ring [%u:%u] existed already\n", bank->bank_number, ring_num);
        return -EBUSY;
    }

    ring->head = 0;
    ring->tail = 0;
    ring->send_seq = 0;
    ring->bank_data = bank;
    /* Now the bank offset is 0 because we get the band's offset */
    ring->bank_offset = 0;
    ring->ring_num = ring_num;
    ring->csr_addr = csr_base_addr;
    ring->message_size = msg_size;
    ring->modulo = modulo;
    ring->ring_size = ring_size_bytes;

    ICP_MEMSET(ring->ring_virt_addr, EMPTY_RING_SIG_BYTE, ring_size_bytes);

    ring->min_resps_per_head_write =
        ((max_space / msg_size) >> 1 > MIN_RESPONSES_PER_HEAD_WRITE)
            ? MIN_RESPONSES_PER_HEAD_WRITE
            : (max_space / msg_size) >> 1;
    ring->max_requests_inflight = num_msgs - 1;

    if (IS_RING_IN_WQ_MODE(ring))
    {
        adf_populate_ring_config(ring, bank, ring_size_cfg);
    }

    adf_io_enable_ring(ring);

    return 0;
}

int32_t adf_init_ring(adf_dev_ring_handle_t *ring,
                      adf_dev_bank_handle_t *bank,
                      uint32_t ring_num,
                      uint32_t *csr_base_addr,
                      uint32_t num_msgs,
                      uint32_t msg_size,
                      int nodeid)
{
    uint32_t modulo = 0;
    uint32_t ring_size_cfg = validateRingSize(num_msgs, msg_size, &modulo);
    uint32_t ring_size_bytes = ICP_ET_SIZE_TO_BYTES(ring_size_cfg);
    int32_t status = 0;

    ring->ring_virt_addr =
        mem_alloc(ring_size_bytes, nodeid, ring_size_bytes, ring);

    if ((NULL == ring->ring_virt_addr))
    {
        ADF_ERROR("Unable to get ringbuf(v:%p) for rings in bank(%lu)\n",
                  ring->ring_virt_addr,
                  (unsigned long)ring->ring_num);
        return -ENOMEM;
    }

    ring->ring_phys_base_addr = qaeVirtToPhysNUMA(ring->ring_virt_addr);
    if (0 == ring->ring_phys_base_addr)
    {
        ADF_ERROR("Unable to get ringbuf(p:%p) for rings in bank(%lu)\n",
                  (void *)ring->ring_phys_base_addr,
                  (unsigned long)ring->ring_num);
        mem_free(&ring->ring_virt_addr, ring);
        return -ENOMEM;
    }

    status = adf_init_ring_internal(
        ring, bank, ring_num, bank->csr_addr, num_msgs, msg_size, nodeid);
    if (status)
    {
        mem_free(&ring->ring_virt_addr, ring);
        return status;
    }

    return 0;
}

int32_t adf_reinit_ring(adf_dev_ring_handle_t *ring,
                        adf_dev_bank_handle_t *bank,
                        uint32_t ring_num,
                        uint32_t *csr_base_addr,
                        uint32_t num_msgs,
                        uint32_t msg_size,
                        int nodeid)
{
    return adf_init_ring_internal(
        ring, bank, ring_num, bank->csr_addr, num_msgs, msg_size, nodeid);
}

static void adf_clean_ring(adf_dev_ring_handle_t *ring)
{
    uint32_t *csr_base_addr = ring->csr_addr;
    ICP_CHECK_FOR_NULL_PARAM_VOID(ring->accel_dev);
    device_type_t deviceType = ring->accel_dev->deviceType;

    adf_io_disable_ring(ring);

    if (IS_RING_IN_WQ_MODE(ring))
    {
        /* Clear CSR configuration */
        if (!IS_QAT_GEN2(deviceType))
        {
            WRITE_CSR_RING_CONFIG_GEN4(
                csr_base_addr, ring->bank_offset, ring->ring_num, 0);
            WRITE_CSR_RING_BASE_GEN4(
                csr_base_addr, ring->bank_offset, ring->ring_num, 0);
        }
        else
        {
            WRITE_CSR_RING_CONFIG(ring->bank_offset, ring->ring_num, 0);
            WRITE_CSR_RING_BASE(ring->bank_offset, ring->ring_num, 0);
        }
    }

    adf_unreserve_ring(ring->bank_data, ring->ring_num);
}

void adf_cleanup_ring(adf_dev_ring_handle_t *ring)
{
    adf_clean_ring(ring);

    if (ring->ring_virt_addr)
    {
        osalMemZeroExplicit(ring->ring_virt_addr, ring->ring_size);
        mem_free(&ring->ring_virt_addr, ring);
    }
}

void adf_reset_ring(adf_dev_ring_handle_t *ring)
{
    uint32_t *csr_addr_shadow = ring->bank_data->csr_addr_shadow;

    adf_clean_ring(ring);

    if (ring->ring_virt_addr)
    {
        ICP_MEMSET(ring->ring_virt_addr, EMPTY_RING_SIG_BYTE, ring->ring_size);
    }
    ring->csr_addr = csr_addr_shadow;
}

int32_t adf_user_get_inflight_requests(adf_dev_ring_handle_t *ring,
                                       uint32_t *maxInflightRequests,
                                       uint32_t *numInflightRequests)
{
    int32_t status = 0;

    status = ICP_MUTEX_LOCK(&ring->user_lock);
    if (OSAL_SUCCESS != status)
    {
        ADF_ERROR("Failed to lock bank with error %d\n", status);
        return -1;
    }

    *numInflightRequests = *ring->in_flight;
    *maxInflightRequests = ring->max_requests_inflight;

    ICP_MUTEX_UNLOCK(&ring->user_lock);

    return status;
}

