github-mirrors/vortex
1
0
Fork 0
mirror of https://github.com/vortexgpgpu/vortex.git synced 2025-04-24 22:07:41 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions
vortex/runtime/opae/vortex.cpp

634 lines
18 KiB
C++
Executable file
Raw Blame History

// Copyright © 2019-2023
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <vortex.h>
#include <utils.h>
#include <malloc.h>
#include "driver.h"
#include <iostream>
#include <stdio.h>
#include <stdlib.h>
#include <cstdlib>
#include <cstring>
#include <uuid/uuid.h>
#include <unistd.h>
#include <assert.h>
#include <cmath>
#include <sstream>
#include <unordered_map>
#include <algorithm>
#include <memory>
#include <list>
#include <VX_config.h>
#include <VX_types.h>
#include <vortex_afu.h>
#ifdef SCOPE
#include "scope.h"
#endif
///////////////////////////////////////////////////////////////////////////////
#define CMD_MEM_READ AFU_IMAGE_CMD_MEM_READ
#define CMD_MEM_WRITE AFU_IMAGE_CMD_MEM_WRITE
#define CMD_RUN AFU_IMAGE_CMD_RUN
#define CMD_DCR_WRITE AFU_IMAGE_CMD_DCR_WRITE
#define MMIO_CMD_TYPE (AFU_IMAGE_MMIO_CMD_TYPE * 4)
#define MMIO_CMD_ARG0 (AFU_IMAGE_MMIO_CMD_ARG0 * 4)
#define MMIO_CMD_ARG1 (AFU_IMAGE_MMIO_CMD_ARG1 * 4)
#define MMIO_CMD_ARG2 (AFU_IMAGE_MMIO_CMD_ARG2 * 4)
#define MMIO_STATUS (AFU_IMAGE_MMIO_STATUS * 4)
#define MMIO_DEV_CAPS (AFU_IMAGE_MMIO_DEV_CAPS * 4)
#define MMIO_ISA_CAPS (AFU_IMAGE_MMIO_ISA_CAPS * 4)
#define MMIO_SCOPE_READ (AFU_IMAGE_MMIO_SCOPE_READ * 4)
#define MMIO_SCOPE_WRITE (AFU_IMAGE_MMIO_SCOPE_WRITE * 4)
#define STATUS_STATE_BITS 8
#define RAM_PAGE_SIZE 4096
#ifndef NDEBUG
#define DBGPRINT(format, ...) do { printf("[VXDRV] " format "", ##__VA_ARGS__); } while (0)
#else
#define DBGPRINT(format, ...) ((void)0)
#endif
#define CHECK_HANDLE(handle, _expr, _cleanup) \
auto handle = _expr; \
if (handle == nullptr) { \
printf("[VXDRV] Error: '%s' returned NULL!\n", #_expr); \
_cleanup \
}
#define CHECK_FPGA_ERR(_expr, _cleanup) \
do { \
auto err = _expr; \
if (err == 0) \
break; \
printf("[VXDRV] Error: '%s' returned %d, %s!\n", #_expr, (int)err, api.fpgaErrStr(err)); \
_cleanup \
} while (false)
#define CHECK_ERR(_expr, _cleanup) \
do { \
auto err = _expr; \
if (err == 0) \
break; \
printf("[VXDRV] Error: '%s' returned %d!\n", #_expr, (int)err); \
_cleanup \
} while (false)
///////////////////////////////////////////////////////////////////////////////
class vx_device {
public:
vx_device() :
staging_wsid(0),
staging_ioaddr(0),
staging_ptr(nullptr),
staging_size(0)
{}
~vx_device() {}
int ensure_staging(uint64_t size) {
size_t asize = aligned_size(size, CACHE_BLOCK_SIZE);
if (staging_size >= asize)
return 0;
if (staging_size != 0) {
// release existing buffer
api.fpgaReleaseBuffer(fpga, staging_wsid);
staging_size = 0;
}
// allocate new buffer
CHECK_FPGA_ERR(api.fpgaPrepareBuffer(fpga, asize, (void**)&staging_ptr, &staging_wsid, 0), {
return -1;
});
// get the physical address of the buffer in the accelerator
CHECK_FPGA_ERR(api.fpgaGetIOAddress(fpga, staging_wsid, &staging_ioaddr), {
api.fpgaReleaseBuffer(fpga, staging_wsid);
return -1;
});
staging_size = asize;
return 0;
}
opae_drv_api_t api;
fpga_handle fpga;
std::shared_ptr<vortex::MemoryAllocator> global_mem;
DeviceConfig dcrs;
uint64_t dev_caps;
uint64_t isa_caps;
uint64_t global_mem_size;
uint64_t staging_wsid;
uint64_t staging_ioaddr;
uint8_t* staging_ptr;
uint64_t staging_size;
};
///////////////////////////////////////////////////////////////////////////////
extern int vx_dev_caps(vx_device_h hdevice, uint32_t caps_id, uint64_t *value) {
if (nullptr == hdevice)
return -1;
auto device = ((vx_device*)hdevice);
switch (caps_id) {
case VX_CAPS_VERSION:
*value = (device->dev_caps >> 0) & 0xff;
break;
case VX_CAPS_NUM_THREADS:
*value = (device->dev_caps >> 8) & 0xff;
break;
case VX_CAPS_NUM_WARPS:
*value = (device->dev_caps >> 16) & 0xff;
break;
case VX_CAPS_NUM_CORES:
*value = (device->dev_caps >> 24) & 0xffff;
break;
case VX_CAPS_CACHE_LINE_SIZE:
*value = CACHE_BLOCK_SIZE;
break;
case VX_CAPS_GLOBAL_MEM_SIZE:
*value = device->global_mem_size;
break;
case VX_CAPS_LOCAL_MEM_SIZE:
*value = 1ull << ((device->dev_caps >> 40) & 0xff);
break;
case VX_CAPS_LOCAL_MEM_ADDR:
*value = LMEM_BASE_ADDR;
break;
case VX_CAPS_ISA_FLAGS:
*value = device->isa_caps;
break;
default:
fprintf(stderr, "[VXDRV] Error: invalid caps id: %d\n", caps_id);
std::abort();
return -1;
}
return 0;
}
extern int vx_dev_open(vx_device_h* hdevice) {
if (nullptr == hdevice)
return -1;
vx_device* device;
fpga_handle accel_handle;
fpga_token accel_token;
fpga_properties filter;
fpga_guid guid;
uint32_t num_matches;
opae_drv_api_t api;
memset(&api, 0, sizeof(opae_drv_api_t));
if (drv_init(&api) !=0) {
return -1;
}
// Set up a filter that will search for an accelerator
CHECK_FPGA_ERR(api.fpgaGetProperties(nullptr, &filter), {
return -1;
});
CHECK_FPGA_ERR(api.fpgaPropertiesSetObjectType(filter, FPGA_ACCELERATOR), {
api.fpgaDestroyProperties(&filter);
return -1;
});
// Add the desired UUID to the filter
std::string s_uuid(AFU_ACCEL_UUID);
std::replace(s_uuid.begin(), s_uuid.end(), '_', '-');
uuid_parse(s_uuid.c_str(), guid);
CHECK_FPGA_ERR(api.fpgaPropertiesSetGUID(filter, guid), {
api.fpgaDestroyProperties(&filter);
return -1;
});
// Do the search across the available FPGA contexts
CHECK_FPGA_ERR(api.fpgaEnumerate(&filter, 1, &accel_token, 1, &num_matches), {
api.fpgaDestroyProperties(&filter);
return -1;
});
// Not needed anymore
CHECK_FPGA_ERR(api.fpgaDestroyProperties(&filter), {
api.fpgaDestroyToken(&accel_token);
return -1;
});
if (num_matches < 1) {
fprintf(stderr, "[VXDRV] Error: accelerator %s not found!\n", AFU_ACCEL_UUID);
api.fpgaDestroyToken(&accel_token);
return -1;
}
// Open accelerator
CHECK_FPGA_ERR(api.fpgaOpen(accel_token, &accel_handle, 0), {
api.fpgaDestroyToken(&accel_token);
return -1;
});
// Done with token
CHECK_FPGA_ERR(api.fpgaDestroyToken(&accel_token), {
api.fpgaClose(accel_handle);
return -1;
});
// allocate device object
device = new vx_device();
if (nullptr == device) {
api.fpgaClose(accel_handle);
return -1;
}
device->api = api;
device->fpga = accel_handle;
{
// retrieve FPGA global memory size
CHECK_FPGA_ERR(api.fpgaPropertiesGetLocalMemorySize(filter, &device->global_mem_size), {
// assume 8GB as default
device->global_mem_size = GLOBAL_MEM_SIZE;
});
// Load ISA CAPS
CHECK_FPGA_ERR(api.fpgaReadMMIO64(device->fpga, 0, MMIO_ISA_CAPS, &device->isa_caps), {
api.fpgaClose(accel_handle);
return -1;
});
// Load device CAPS
CHECK_FPGA_ERR(api.fpgaReadMMIO64(device->fpga, 0, MMIO_DEV_CAPS, &device->dev_caps), {
api.fpgaClose(accel_handle);
return -1;
});
}
device->global_mem = std::make_shared<vortex::MemoryAllocator>(
ALLOC_BASE_ADDR, ALLOC_MAX_ADDR - ALLOC_BASE_ADDR, RAM_PAGE_SIZE, CACHE_BLOCK_SIZE);
#ifdef SCOPE
{
scope_callback_t callback;
callback.registerWrite = [](vx_device_h hdevice, uint64_t value)->int {
auto device = (vx_device*)hdevice;
return device->api.fpgaWriteMMIO64(device->fpga, 0, MMIO_SCOPE_WRITE, value);
};
callback.registerRead = [](vx_device_h hdevice, uint64_t* value)->int {
auto device = (vx_device*)hdevice;
return device->api.fpgaReadMMIO64(device->fpga, 0, MMIO_SCOPE_READ, value);
};
int ret = vx_scope_start(&callback, device, 0, -1);
if (ret != 0) {
api.fpgaClose(accel_handle);
return ret;
}
}
#endif
int err = dcr_initialize(device);
if (err != 0) {
delete device;
return err;
}
#ifdef DUMP_PERF_STATS
perf_add_device(device);
#endif
*hdevice = device;
return 0;
}
extern int vx_dev_close(vx_device_h hdevice) {
if (nullptr == hdevice)
return -1;
auto device = ((vx_device*)hdevice);
auto& api = device->api;
#ifdef SCOPE
vx_scope_stop(hdevice);
#endif
#ifdef DUMP_PERF_STATS
perf_remove_device(hdevice);
#endif
// release staging buffer
if (device->staging_size != 0) {
api.fpgaReleaseBuffer(device->fpga, device->staging_wsid);
device->staging_size = 0;
}
// close the device
api.fpgaClose(device->fpga);
delete device;
drv_close();
return 0;
}
extern int vx_mem_alloc(vx_device_h hdevice, uint64_t size, uint64_t* dev_addr) {
if (nullptr == hdevice
|| nullptr == dev_addr
|| 0 == size)
return -1;
auto device = ((vx_device*)hdevice);
return device->global_mem->allocate(size, dev_addr);
}
extern int vx_mem_free(vx_device_h hdevice, uint64_t dev_addr) {
if (nullptr == hdevice)
return -1;
if (0 == dev_addr)
return 0;
auto device = ((vx_device*)hdevice);
return device->global_mem->release(dev_addr);
}
extern int vx_mem_info(vx_device_h hdevice, uint64_t* mem_free, uint64_t* mem_used) {
if (nullptr == hdevice)
return -1;
auto device = ((vx_device*)hdevice);
if (mem_free)
*mem_free = device->global_mem->free();
if (mem_used)
*mem_used = device->global_mem->allocated();
return 0;
}
extern int vx_copy_to_dev(vx_device_h hdevice, uint64_t dev_addr, const void* host_ptr, uint64_t size) {
if (nullptr == hdevice)
return -1;
auto device = (vx_device*)hdevice;
auto& api = device->api;
DBGPRINT("COPY_TO_DEV: dev_addr=0x%lx, host_addr=0x%lx, size=%ld\n", dev_addr, (uintptr_t)host_ptr, size);
if (device->ensure_staging(size) != 0)
return -1;
uint64_t asize = aligned_size(size, CACHE_BLOCK_SIZE);
// check alignment
if (!is_aligned(dev_addr, CACHE_BLOCK_SIZE))
return -1;
// bound checking
if (dev_addr + asize > device->global_mem_size)
return -1;
// ensure ready for new command
if (vx_ready_wait(hdevice, VX_MAX_TIMEOUT) != 0)
return -1;
// update staging buffer
memcpy(device->staging_ptr, host_ptr, size);
auto ls_shift = (int)std::log2(CACHE_BLOCK_SIZE);
CHECK_FPGA_ERR(api.fpgaWriteMMIO64(device->fpga, 0, MMIO_CMD_ARG0, device->staging_ioaddr >> ls_shift), {
return -1;
});
CHECK_FPGA_ERR(api.fpgaWriteMMIO64(device->fpga, 0, MMIO_CMD_ARG1, dev_addr >> ls_shift), {
return -1;
});
CHECK_FPGA_ERR(api.fpgaWriteMMIO64(device->fpga, 0, MMIO_CMD_ARG2, asize >> ls_shift), {
return -1;
});
CHECK_FPGA_ERR(api.fpgaWriteMMIO64(device->fpga, 0, MMIO_CMD_TYPE, CMD_MEM_WRITE), {
return -1;
});
// Wait for the write operation to finish
if (vx_ready_wait(hdevice, VX_MAX_TIMEOUT) != 0)
return -1;
return 0;
}
extern int vx_copy_from_dev(vx_device_h hdevice, void* host_ptr, uint64_t dev_addr, uint64_t size) {
if (nullptr == hdevice)
return -1;
auto device = (vx_device*)hdevice;
auto& api = device->api;
DBGPRINT("COPY_FROM_DEV: dev_addr=0x%lx, host_addr=0x%lx, size=%ld\n", dev_addr, (uintptr_t)host_ptr, asize);
if (device->ensure_staging(size) != 0)
return -1;
uint64_t asize = aligned_size(size, CACHE_BLOCK_SIZE);
// check alignment
if (!is_aligned(dev_addr, CACHE_BLOCK_SIZE))
return -1;
// bound checking
if (dev_addr + asize > device->global_mem_size)
return -1;
// Ensure ready for new command
if (vx_ready_wait(hdevice, VX_MAX_TIMEOUT) != 0)
return -1;
auto ls_shift = (int)std::log2(CACHE_BLOCK_SIZE);
CHECK_FPGA_ERR(api.fpgaWriteMMIO64(device->fpga, 0, MMIO_CMD_ARG0, device->staging_ioaddr >> ls_shift), {
return -1;
});
CHECK_FPGA_ERR(api.fpgaWriteMMIO64(device->fpga, 0, MMIO_CMD_ARG1, dev_addr >> ls_shift), {
return -1;
});
CHECK_FPGA_ERR(api.fpgaWriteMMIO64(device->fpga, 0, MMIO_CMD_ARG2, asize >> ls_shift), {
return -1;
});
CHECK_FPGA_ERR(api.fpgaWriteMMIO64(device->fpga, 0, MMIO_CMD_TYPE, CMD_MEM_READ), {
return -1;
});
// wait for the write operation to finish
if (vx_ready_wait(hdevice, VX_MAX_TIMEOUT) != 0)
return -1;
// read staging buffer
memcpy(host_ptr, device->staging_ptr, size);
return 0;
}
extern int vx_start(vx_device_h hdevice, uint64_t krnl_addr, uint64_t args_addr) {
if (nullptr == hdevice)
return -1;
auto device = ((vx_device*)hdevice);
auto& api = device->api;
DBGPRINT("START: krnl_addr=0x%lx, args_addr=0x%lx\n", krnl_addr, args_addr);
// ensure ready for new command
if (vx_ready_wait(hdevice, VX_MAX_TIMEOUT) != 0)
return -1;
// set kernel info
CHECK_ERR(vx_dcr_write(hdevice, VX_DCR_BASE_STARTUP_ADDR0, krnl_addr & 0xffffffff), {
return -1;
});
CHECK_ERR(vx_dcr_write(hdevice, VX_DCR_BASE_STARTUP_ADDR0, krnl_addr >> 32), {
return -1;
});
CHECK_ERR(vx_dcr_write(hdevice, VX_DCR_BASE_STARTUP_ARG0, args_addr & 0xffffffff), {
return -1;
});
CHECK_ERR(vx_dcr_write(hdevice, VX_DCR_BASE_STARTUP_ARG0, args_addr >> 32), {
return -1;
});
// start execution
CHECK_FPGA_ERR(api.fpgaWriteMMIO64(device->fpga, 0, MMIO_CMD_TYPE, CMD_RUN), {
return -1;
});
return 0;
}
extern int vx_ready_wait(vx_device_h hdevice, uint64_t timeout) {
if (nullptr == hdevice)
return -1;
std::unordered_map<uint32_t, std::stringstream> print_bufs;
auto device = ((vx_device*)hdevice);
auto& api = device->api;
DBGPRINT("%s\n", "WAIT");
struct timespec sleep_time;
sleep_time.tv_sec = 0;
sleep_time.tv_nsec = 1000000;
// to milliseconds
uint64_t sleep_time_ms = (sleep_time.tv_sec * 1000) + (sleep_time.tv_nsec / 1000000);
for (;;) {
uint64_t status;
CHECK_FPGA_ERR(api.fpgaReadMMIO64(device->fpga, 0, MMIO_STATUS, &status), {
return -1;
});
// check for console data
uint32_t cout_data = status >> STATUS_STATE_BITS;
if (cout_data & 0x1) {
// retrieve console data
do {
char cout_char = (cout_data >> 1) & 0xff;
uint32_t cout_tid = (cout_data >> 9) & 0xff;
auto& ss_buf = print_bufs[cout_tid];
ss_buf << cout_char;
if (cout_char == '\n') {
std::cout << std::dec << "#" << cout_tid << ": " << ss_buf.str() << std::flush;
ss_buf.str("");
}
CHECK_FPGA_ERR(api.fpgaReadMMIO64(device->fpga, 0, MMIO_STATUS, &status), {
return -1;
});
cout_data = status >> STATUS_STATE_BITS;
} while (cout_data & 0x1);
}
uint32_t state = status & ((1 << STATUS_STATE_BITS)-1);
if (0 == state || 0 == timeout) {
for (auto& buf : print_bufs) {
auto str = buf.second.str();
if (!str.empty()) {
std::cout << "#" << buf.first << ": " << str << std::endl;
}
}
if (state != 0) {
fprintf(stdout, "[VXDRV] ready-wait timed out: state=%d\n", state);
}
break;
}
nanosleep(&sleep_time, nullptr);
timeout -= sleep_time_ms;
};
return 0;
}
extern int vx_dcr_read(vx_device_h hdevice, uint32_t addr, uint32_t* value) {
if (nullptr == hdevice)
return -1;
auto device = (vx_device*)hdevice;
*value = device->dcrs.read(addr);
DBGPRINT("DCR_READ: addr=0x%x, value=0x%x\n", addr, *value);
return 0;
}
extern int vx_dcr_write(vx_device_h hdevice, uint32_t addr, uint32_t value) {
if (nullptr == hdevice)
return -1;
auto device = ((vx_device*)hdevice);
auto& api = device->api;
DBGPRINT("DCR_WRITE: addr=0x%x, value=0x%x\n", addr, value);
// Ensure ready for new command
if (vx_ready_wait(hdevice, -1) != 0)
return -1;
// write DCR value
CHECK_FPGA_ERR(api.fpgaWriteMMIO64(device->fpga, 0, MMIO_CMD_ARG0, addr), {
return -1;
});
CHECK_FPGA_ERR(api.fpgaWriteMMIO64(device->fpga, 0, MMIO_CMD_ARG1, value), {
return -1;
});
CHECK_FPGA_ERR(api.fpgaWriteMMIO64(device->fpga, 0, MMIO_CMD_TYPE, CMD_DCR_WRITE), {
return -1;
});
device->dcrs.write(addr, value);
return 0;
}
Reference in a new issue View git blame Copy permalink