RedBear-OS/local/recipes/gpu/amdgpu/source/redox_stubs.c

#include "redox_glue.h"

#include <fcntl.h>
#include <errno.h>
#include <stdatomic.h>
#include <poll.h>
#include <sys/mman.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <unistd.h>

unsigned long jiffies;

struct redox_mapped_region {
    void *addr;
    size_t size;
    int fd;
    struct redox_mapped_region *next;
};

struct redox_irq_entry {
    unsigned int irq;
    irq_handler_t handler;
    void *dev_id;
    int fd;
    pthread_t thread;
    bool active;
    struct redox_irq_entry *next;
};

struct redox_pci_region {
    struct pci_dev *pdev;
    unsigned int bar;
    void *addr;
    size_t size;
    bool claimed;
    struct redox_pci_region *next;
};

static pthread_mutex_t g_region_lock = PTHREAD_MUTEX_INITIALIZER;
static struct redox_mapped_region *g_regions;
static pthread_mutex_t g_irq_lock = PTHREAD_MUTEX_INITIALIZER;
static struct redox_irq_entry *g_irqs;
static pthread_mutex_t g_pci_region_lock = PTHREAD_MUTEX_INITIALIZER;
static struct redox_pci_region *g_pci_regions;

static void redox_jiffies_advance(unsigned long delta)
{
    __sync_add_and_fetch(&jiffies, delta);
}

void *vmalloc(unsigned long size)
{
    return malloc((size_t)size);
}

void vfree(const void *addr)
{
    free((void *)addr);
}

void *krealloc(const void *ptr, size_t new_size, unsigned int flags)
{
    (void)flags;
    return realloc((void *)ptr, new_size);
}

static void redox_track_region(void *addr, size_t size, int fd)
{
    struct redox_mapped_region *region = malloc(sizeof(*region));
    if (!region) {
        if (fd >= 0) {
            close(fd);
        }
        return;
    }

    region->addr = addr;
    region->size = size;
    region->fd = fd;

    pthread_mutex_lock(&g_region_lock);
    region->next = g_regions;
    g_regions = region;
    pthread_mutex_unlock(&g_region_lock);
}

static struct redox_mapped_region *redox_untrack_region(const void *addr)
{
    struct redox_mapped_region *prev = NULL;
    struct redox_mapped_region *cur;

    pthread_mutex_lock(&g_region_lock);
    cur = g_regions;
    while (cur) {
        if (cur->addr == addr) {
            if (prev) {
                prev->next = cur->next;
            } else {
                g_regions = cur->next;
            }
            pthread_mutex_unlock(&g_region_lock);
            return cur;
        }
        prev = cur;
        cur = cur->next;
    }
    pthread_mutex_unlock(&g_region_lock);
    return NULL;
}

void __iomem *redox_ioremap(phys_addr_t offset, size_t size)
{
    int fd = open("/scheme/memory/physical", O_RDWR);
    void *addr;

    if (fd >= 0) {
        addr = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, (off_t)offset);
        if (addr != MAP_FAILED) {
            redox_track_region(addr, size, fd);
            return addr;
        }
        close(fd);
    }

    pr_err("ioremap failed for %#llx (%zu bytes): %s\n",
           (unsigned long long)offset, size, strerror(errno));
    return NULL;
}

void redox_iounmap(void __iomem *addr)
{
    struct redox_mapped_region *region;

    if (!addr) {
        return;
    }

    region = redox_untrack_region(addr);
    if (!region) {
        return;
    }

    munmap(region->addr, region->size);
    if (region->fd >= 0) {
        close(region->fd);
    }
    free(region);
}

void redox_iowrite32(u32 val, void __iomem *addr)
{
    *(volatile u32 *)addr = val;
}

u32 redox_ioread32(const void __iomem *addr)
{
    return *(volatile const u32 *)addr;
}

void redox_iowrite16(u16 val, void __iomem *addr)
{
    *(volatile u16 *)addr = val;
}

u16 redox_ioread16(const void __iomem *addr)
{
    return *(volatile const u16 *)addr;
}

void redox_iowrite8(u8 val, void __iomem *addr)
{
    *(volatile u8 *)addr = val;
}

u8 redox_ioread8(const void __iomem *addr)
{
    return *(volatile const u8 *)addr;
}

void redox_mmio_write32(void *base, u32 offset, u32 val)
{
    if (!base) {
        return;
    }
    *(volatile u32 *)((u8 *)base + offset) = val;
}

u32 redox_mmio_read32(void *base, u32 offset)
{
    if (!base) {
        return 0;
    }
    return *(volatile u32 *)((u8 *)base + offset);
}

void *redox_dma_alloc_coherent(size_t size, dma_addr_t *dma_handle)
{
    void *ptr = NULL;

    if (posix_memalign(&ptr, PAGE_SIZE, PAGE_ALIGN(size)) != 0) {
        return NULL;
    }

    memset(ptr, 0, PAGE_ALIGN(size));
    if (dma_handle) {
        *dma_handle = (dma_addr_t)(uintptr_t)ptr;
    }
    return ptr;
}

void redox_dma_free_coherent(size_t size, void *vaddr, dma_addr_t dma_handle)
{
    (void)size;
    (void)dma_handle;
    free(vaddr);
}

/*
 * PCI device state — populated by the Rust side via redox_pci_set_device_info()
 * before amdgpu_redox_init() is called.  redox_pci_find_amd_gpu() returns a
 * pointer to this struct, or NULL if the device info has not been set yet.
 */
static struct pci_dev g_pci_dev;
static int g_pci_dev_populated;

#define REDOX_MAX_FIRMWARE_BYTES (64U * 1024U * 1024U)

void redox_pci_set_device_info(u16 vendor, u16 device,
                                u8 bus_number, u8 dev_number,
                                u8 func_number, u8 revision, u32 irq,
                                u64 bar0_addr, u64 bar0_size,
                                u64 bar2_addr, u64 bar2_size)
{
    memset(&g_pci_dev, 0, sizeof(g_pci_dev));
    g_pci_dev.vendor = vendor;
    g_pci_dev.device_id = device;
    g_pci_dev.bus_number = bus_number;
    g_pci_dev.dev_number = dev_number;
    g_pci_dev.func_number = func_number;
    g_pci_dev.revision = revision;
    g_pci_dev.irq = irq;
    g_pci_dev.resource_start[0] = (phys_addr_t)bar0_addr;
    g_pci_dev.resource_len[0] = bar0_size;
    g_pci_dev.resource_flags[0] = IORESOURCE_MEM;
    g_pci_dev.resource_start[2] = (phys_addr_t)bar2_addr;
    g_pci_dev.resource_len[2] = bar2_size;
    g_pci_dev.resource_flags[2] = IORESOURCE_MEM;
    g_pci_dev.driver_data = NULL;
    memset(&g_pci_dev.device_obj, 0, sizeof(g_pci_dev.device_obj));
    g_pci_dev.enabled = false;
    g_pci_dev.refcount = 1;
    g_pci_dev.mmio_base = NULL;
    g_pci_dev.is_amdgpu = 1;
    g_pci_dev_populated = 1;

    printk("PCI device info set: %02x:%02x.%u vendor=%#06x device=%#06x rev=%#04x irq=%u "
           "bar0=%#llx+%#llx bar2=%#llx+%#llx\n",
           bus_number, dev_number, func_number,
           vendor, device, revision, irq,
           (unsigned long long)bar0_addr, (unsigned long long)bar0_size,
           (unsigned long long)bar2_addr, (unsigned long long)bar2_size);
}

struct pci_dev *redox_pci_find_amd_gpu(void)
{
    if (!g_pci_dev_populated) {
        pr_err("redox_pci_find_amd_gpu: device info not set — "
               "call redox_pci_set_device_info() first\n");
        return NULL;
    }

    return &g_pci_dev;
}

void redox_pci_dev_put(struct pci_dev *pdev)
{
    if (!pdev) {
        return;
    }

    if (__sync_sub_and_fetch(&pdev->refcount, 1) == 0) {
        redox_pci_release_regions(pdev);
        free(pdev);
    }
}

int redox_pci_enable_device(struct pci_dev *pdev)
{
    return pdev ? 0 : -ENODEV;
}

void redox_pci_set_master(struct pci_dev *pdev)
{
    if (!pdev) {
        return;
    }

    pdev->command |= (u16)(1U << 2);
}

int redox_pci_request_regions(struct pci_dev *pdev, const char *name)
{
    unsigned int bar;
    (void)name;

    if (!pdev) {
        return -ENODEV;
    }

    pthread_mutex_lock(&g_pci_region_lock);
    for (bar = 0; bar < 6; ++bar) {
        struct redox_pci_region *region;
        if (!(pdev->resource_flags[bar] & IORESOURCE_MEM) || pdev->resource_len[bar] == 0) {
            continue;
        }
        for (region = g_pci_regions; region != NULL; region = region->next) {
            if (region->claimed && region->pdev != pdev) {
                u64 a0 = pdev->resource_start[bar];
                u64 a1 = a0 + pdev->resource_len[bar] - 1;
                u64 b0 = (u64)(uintptr_t)region->addr;
                u64 b1 = b0 + region->size - 1;
                if (!(a1 < b0 || b1 < a0)) {
                    pthread_mutex_unlock(&g_pci_region_lock);
                    return -EBUSY;
                }
            }
        }
        region = calloc(1, sizeof(*region));
        if (!region) {
            pthread_mutex_unlock(&g_pci_region_lock);
            return -ENOMEM;
        }
        region->pdev = pdev;
        region->bar = bar;
        region->size = (size_t)pdev->resource_len[bar];
        region->addr = redox_ioremap((phys_addr_t)pdev->resource_start[bar], region->size);
        if (!region->addr) {
            free(region);
            pthread_mutex_unlock(&g_pci_region_lock);
            return -ENOMEM;
        }
        region->claimed = true;
        region->next = g_pci_regions;
        g_pci_regions = region;
    }
    pthread_mutex_unlock(&g_pci_region_lock);
    return 0;
}

void redox_pci_release_regions(struct pci_dev *pdev)
{
    struct redox_pci_region **link;
    if (!pdev) {
        return;
    }

    pthread_mutex_lock(&g_pci_region_lock);
    link = &g_pci_regions;
    while (*link) {
        struct redox_pci_region *region = *link;
        if (region->pdev == pdev) {
            *link = region->next;
            pthread_mutex_unlock(&g_pci_region_lock);
            if (region->addr) {
                redox_iounmap((void __iomem *)region->addr);
            }
            free(region);
            pthread_mutex_lock(&g_pci_region_lock);
            link = &g_pci_regions;
            continue;
        }
        link = &region->next;
    }
    pthread_mutex_unlock(&g_pci_region_lock);
}

int redox_request_firmware(const struct firmware **fw, const char *name, void *dev)
{
    char path[512];
    int fd;
    struct stat st;
    struct firmware *image;
    u8 *data;
    ssize_t nread;

    (void)dev;
    if (!fw || !name) {
        return -EINVAL;
    }

    snprintf(path, sizeof(path), "/scheme/firmware/amdgpu/%s", name);
    fd = open(path, O_RDONLY);
    if (fd < 0) {
        return -ENOENT;
    }

    if (fstat(fd, &st) != 0 || st.st_size < 0) {
        close(fd);
        return -EIO;
    }

    if ((unsigned long long)st.st_size > REDOX_MAX_FIRMWARE_BYTES) {
        close(fd);
        return -EFBIG;
    }

    image = calloc(1, sizeof(*image));
    data = malloc((size_t)st.st_size);
    if (!image || !data) {
        free(image);
        free(data);
        close(fd);
        return -ENOMEM;
    }

    nread = read(fd, data, (size_t)st.st_size);
    close(fd);
    if (nread != st.st_size) {
        free(image);
        free(data);
        return -EIO;
    }

    image->size = (size_t)st.st_size;
    image->data = data;
    *fw = image;
    return 0;
}

void redox_release_firmware(const struct firmware *fw)
{
    struct firmware *owned = (struct firmware *)fw;

    if (!owned) {
        return;
    }

    free((void *)owned->data);
    free(owned);
}

static void *redox_irq_thread_main(void *arg)
{
    struct redox_irq_entry *entry = arg;
    unsigned char buf[64];

    pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL);
    pthread_setcanceltype(PTHREAD_CANCEL_DEFERRED, NULL);

    while (entry->active) {
        ssize_t n = read(entry->fd, buf, sizeof(buf));
        if (n < 0) {
            if (errno == EINTR) {
                continue;
            }
            break;
        }
        if (n == 0) {
            break;
        }
        entry->handler((int)entry->irq, entry->dev_id);
    }

    return NULL;
}

int redox_request_irq(unsigned int irq, irq_handler_t handler, unsigned long flags, const char *name, void *dev)
{
    char path[128];
    int fd;
    struct redox_irq_entry *entry;

    (void)flags;
    (void)name;
    if (!handler) {
        return -EINVAL;
    }

    snprintf(path, sizeof(path), "/scheme/irq/%u", irq);
    fd = open(path, O_RDWR);
    if (fd < 0) {
        return -ENOENT;
    }

    entry = calloc(1, sizeof(*entry));
    if (!entry) {
        close(fd);
        return -ENOMEM;
    }

    entry->irq = irq;
    entry->handler = handler;
    entry->dev_id = dev;
    entry->fd = fd;
    entry->active = true;

    pthread_mutex_lock(&g_irq_lock);
    entry->next = g_irqs;
    g_irqs = entry;
    pthread_mutex_unlock(&g_irq_lock);

    if (pthread_create(&entry->thread, NULL, redox_irq_thread_main, entry) != 0) {
        pthread_mutex_lock(&g_irq_lock);
        if (g_irqs == entry) {
            g_irqs = entry->next;
        } else {
            struct redox_irq_entry *cur = g_irqs;
            while (cur && cur->next != entry) {
                cur = cur->next;
            }
            if (cur) {
                cur->next = entry->next;
            }
        }
        pthread_mutex_unlock(&g_irq_lock);
        close(fd);
        free(entry);
        return -EFAULT;
    }

    return 0;
}

void redox_free_irq(unsigned int irq, void *dev_id)
{
    struct redox_irq_entry **link;
    struct redox_irq_entry *entry = NULL;

    pthread_mutex_lock(&g_irq_lock);
    link = &g_irqs;
    while (*link) {
        if ((*link)->irq == irq && (*link)->dev_id == dev_id) {
            entry = *link;
            *link = entry->next;
            entry->active = false;
            break;
        }
        link = &(*link)->next;
    }
    pthread_mutex_unlock(&g_irq_lock);

    if (!entry) {
        return;
    }

    pthread_cancel(entry->thread);
    pthread_join(entry->thread, NULL);
    close(entry->fd);
    free(entry);
}

void msleep(unsigned int msecs)
{
    struct timespec ts;

    ts.tv_sec = msecs / 1000U;
    ts.tv_nsec = (long)(msecs % 1000U) * 1000000L;
    nanosleep(&ts, NULL);
    redox_jiffies_advance(msecs_to_jiffies(msecs));
}

void udelay(unsigned long usecs)
{
    struct timespec ts;

    ts.tv_sec = usecs / 1000000UL;
    ts.tv_nsec = (long)(usecs % 1000000UL) * 1000L;
    nanosleep(&ts, NULL);
    redox_jiffies_advance(usecs_to_jiffies((unsigned int)usecs));
}

void mdelay(unsigned long msecs)
{
    msleep((unsigned int)msecs);
}

unsigned long msecs_to_jiffies(unsigned int msecs)
{
    return (unsigned long)msecs;
}

unsigned long usecs_to_jiffies(unsigned int usecs)
{
    return (unsigned long)DIV_ROUND_UP(usecs, 1000U);
}

struct redox_pm_state {
    struct device *dev;
    atomic_int usage_count;
    bool enabled;
    bool allowed;
    bool active;
    bool ignore_children;
    bool no_pm;
    struct redox_pm_state *next;
};

static pthread_mutex_t g_pm_lock = PTHREAD_MUTEX_INITIALIZER;
static struct redox_pm_state *g_pm_states;

static struct redox_pm_state *redox_pm_find_state(struct device *dev)
{
    struct redox_pm_state *state;

    for (state = g_pm_states; state != NULL; state = state->next) {
        if (state->dev == dev) {
            return state;
        }
    }
    return NULL;
}

static struct redox_pm_state *redox_pm_get_state(struct device *dev)
{
    struct redox_pm_state *state = redox_pm_find_state(dev);
    struct pci_dev *pdev;

    if (state != NULL || dev == NULL) {
        return state;
    }

    state = kzalloc(sizeof(*state), 0);
    if (state == NULL) {
        return NULL;
    }

    state->dev = dev;
    atomic_init(&state->usage_count, 0);
    pdev = dev->pci_dev;
    if (pdev != NULL) {
        state->no_pm = pci_has_quirk(pdev, PCI_QUIRK_NO_PM);
    }

    pthread_mutex_lock(&g_pm_lock);
    state->next = g_pm_states;
    g_pm_states = state;
    pthread_mutex_unlock(&g_pm_lock);

    return state;
}

static bool redox_pm_blocked(struct redox_pm_state *state)
{
    return state == NULL || state->no_pm;
}

int pm_runtime_get_sync(struct device *dev)
{
    struct redox_pm_state *state = redox_pm_get_state(dev);

    if (redox_pm_blocked(state)) {
        return 1;
    }

    pthread_mutex_lock(&g_pm_lock);
    atomic_fetch_add_explicit(&state->usage_count, 1, memory_order_relaxed);
    state->active = true;
    int usage = atomic_load_explicit(&state->usage_count, memory_order_relaxed);
    pthread_mutex_unlock(&g_pm_lock);

    dev_info(dev, "runtime PM get_sync: usage=%d\n", usage);
    return usage;
}

int pm_runtime_get_noresume(struct device *dev)
{
    struct redox_pm_state *state = redox_pm_get_state(dev);

    if (redox_pm_blocked(state)) {
        return 1;
    }

    pthread_mutex_lock(&g_pm_lock);
    atomic_fetch_add_explicit(&state->usage_count, 1, memory_order_relaxed);
    int usage = atomic_load_explicit(&state->usage_count, memory_order_relaxed);
    pthread_mutex_unlock(&g_pm_lock);

    dev_info(dev, "runtime PM get_noresume: usage=%d\n", usage);
    return usage;
}

int pm_runtime_put_autosuspend(struct device *dev)
{
    struct redox_pm_state *state = redox_pm_get_state(dev);

    if (redox_pm_blocked(state)) {
        return 0;
    }

    pthread_mutex_lock(&g_pm_lock);
    if (atomic_load_explicit(&state->usage_count, memory_order_relaxed) > 0) {
        atomic_fetch_sub_explicit(&state->usage_count, 1, memory_order_relaxed);
    }
    int usage = atomic_load_explicit(&state->usage_count, memory_order_relaxed);
    if (usage == 0 && state->allowed && state->enabled) {
        state->active = false;
    }
    pthread_mutex_unlock(&g_pm_lock);

    dev_info(dev, "runtime PM put_autosuspend: usage=%d active=%d\n",
             usage, state->active ? 1 : 0);
    return usage;
}

int pm_runtime_put_noidle(struct device *dev)
{
    struct redox_pm_state *state = redox_pm_get_state(dev);

    if (redox_pm_blocked(state)) {
        return 0;
    }

    pthread_mutex_lock(&g_pm_lock);
    if (atomic_load_explicit(&state->usage_count, memory_order_relaxed) > 0) {
        atomic_fetch_sub_explicit(&state->usage_count, 1, memory_order_relaxed);
    }
    int usage = atomic_load_explicit(&state->usage_count, memory_order_relaxed);
    pthread_mutex_unlock(&g_pm_lock);

    dev_info(dev, "runtime PM put_noidle: usage=%d\n", usage);
    return usage;
}

int pm_runtime_idle(struct device *dev)
{
    struct redox_pm_state *state = redox_pm_get_state(dev);

    if (redox_pm_blocked(state)) {
        return 0;
    }

    pthread_mutex_lock(&g_pm_lock);
    if (atomic_load_explicit(&state->usage_count, memory_order_relaxed) == 0 && state->allowed && state->enabled) {
        state->active = false;
    }
    int usage = atomic_load_explicit(&state->usage_count, memory_order_relaxed);
    pthread_mutex_unlock(&g_pm_lock);

    dev_info(dev, "runtime PM idle: active=%d\n", state->active ? 1 : 0);
    return usage;
}

int pm_runtime_set_active(struct device *dev)
{
    struct redox_pm_state *state = redox_pm_get_state(dev);

    if (redox_pm_blocked(state)) {
        return 0;
    }

    pthread_mutex_lock(&g_pm_lock);
    state->active = true;
    if (atomic_load_explicit(&state->usage_count, memory_order_relaxed) < 1) {
        atomic_store_explicit(&state->usage_count, 1, memory_order_relaxed);
    }
    int usage = atomic_load_explicit(&state->usage_count, memory_order_relaxed);
    pthread_mutex_unlock(&g_pm_lock);

    dev_info(dev, "runtime PM set_active\n");
    return usage;
}

int pm_runtime_enable(struct device *dev)
{
    struct redox_pm_state *state = redox_pm_get_state(dev);

    if (redox_pm_blocked(state)) {
        return 0;
    }

    pthread_mutex_lock(&g_pm_lock);
    state->enabled = true;
    pthread_mutex_unlock(&g_pm_lock);

    dev_info(dev, "runtime PM enabled\n");
    return 0;
}

int pm_runtime_disable(struct device *dev)
{
    struct redox_pm_state *state = redox_pm_get_state(dev);

    if (state == NULL) {
        return 0;
    }

    pthread_mutex_lock(&g_pm_lock);
    state->enabled = false;
    pthread_mutex_unlock(&g_pm_lock);

    dev_info(dev, "runtime PM disabled\n");
    return 0;
}

int pm_runtime_allow(struct device *dev)
{
    struct redox_pm_state *state = redox_pm_get_state(dev);

    if (state == NULL) {
        return 0;
    }

    pthread_mutex_lock(&g_pm_lock);
    state->allowed = true;
    pthread_mutex_unlock(&g_pm_lock);

    dev_info(dev, "runtime PM allowed\n");
    return 0;
}

int pm_runtime_forbid(struct device *dev)
{
    struct redox_pm_state *state = redox_pm_get_state(dev);

    if (state == NULL) {
        return 0;
    }

    pthread_mutex_lock(&g_pm_lock);
    state->allowed = false;
    pthread_mutex_unlock(&g_pm_lock);

    dev_info(dev, "runtime PM forbidden\n");
    return 0;
}

void pm_suspend_ignore_children(struct device *dev, int enable)
{
    struct redox_pm_state *state = redox_pm_get_state(dev);

    if (state == NULL) {
        return;
    }

    pthread_mutex_lock(&g_pm_lock);
    state->ignore_children = (bool)enable;
    pthread_mutex_unlock(&g_pm_lock);

    dev_info(dev, "runtime PM ignore_children=%d\n", enable ? 1 : 0);
}