opae_sim buffer index allocation bug fix

driver/opae/vlsim/fpga.cpp

@@ -79,16 +79,6 @@ extern fpga_result fpgaReadMMIO64(fpga_handle handle, uint32_t mmio_num, uint64_
   return FPGA_OK;
 }
 
-extern fpga_result fpgaFlush(fpga_handle handle) {
-  if (NULL == handle) 
-    return FPGA_INVALID_PARAM;
-
-  auto sim = reinterpret_cast<opae_sim*>(handle);
-  sim->flush();
-
-  return FPGA_OK;
-}
-
 extern const char *fpgaErrStr(fpga_result e) {
   return "";
 }

driver/opae/vlsim/fpga.h

@@ -39,8 +39,6 @@ fpga_result fpgaWriteMMIO64(fpga_handle handle, uint32_t mmio_num, uint64_t offs
 
 fpga_result fpgaReadMMIO64(fpga_handle handle, uint32_t mmio_num, uint64_t offset, uint64_t *value);
 
-fpga_result fpgaFlush(fpga_handle handle);
-
 const char *fpgaErrStr(fpga_result e);
 
 #ifdef __cplusplus

driver/opae/vlsim/opae_sim.cpp

@@ -34,7 +34,27 @@ double sc_time_stamp() {
   return timestamp;
 }
 
-opae_sim::opae_sim() {  
+static void *__aligned_malloc(size_t alignment, size_t size) {
+  // reserve margin for alignment and storing of unaligned address
+  size_t margin = (alignment-1) + sizeof(void*);
+  void *unaligned_addr = malloc(size + margin);
+  void **aligned_addr = (void**)((uintptr_t)(((uint8_t*)unaligned_addr) + margin) & ~(alignment-1));
+  aligned_addr[-1] = unaligned_addr;
+  return aligned_addr;
+}
+
+static void __aligned_free(void *ptr) {
+  // retreive the stored unaligned address and use it to free the allocation
+  void* unaligned_addr = ((void**)ptr)[-1];
+  free(unaligned_addr);
+}
+
+///////////////////////////////////////////////////////////////////////////////
+
+opae_sim::opae_sim() 
+  : stop_(false)
+  , host_buffer_ids_(0) 
+{  
   // force random values for unitialized signals  
   Verilated::randReset(VERILATOR_RESET_VALUE);
   Verilated::randSeed(50);
@@ -55,7 +75,6 @@ opae_sim::opae_sim() {
   this->reset();
 
   // launch execution thread
-  stop_ = false;
   future_ = std::async(std::launch::async, [&]{                   
       while (!stop_) {
           std::lock_guard<std::mutex> guard(mutex_);
@@ -72,24 +91,12 @@ opae_sim::~opae_sim() {
 #ifdef VCD_OUTPUT
   trace_->close();
 #endif  
+  for (auto& buffer : host_buffers_) {
+    __aligned_free(buffer.second.data);
+  }   
   delete vortex_afu_;
 }
 
-static void *__aligned_malloc(size_t alignment, size_t size) {
-  // reserve margin for alignment and storing of unaligned address
-  size_t margin = (alignment-1) + sizeof(void*);
-  void *unaligned_addr = malloc(size + margin);
-  void **aligned_addr = (void**)((uintptr_t)(((uint8_t*)unaligned_addr) + margin) & ~(alignment-1));
-  aligned_addr[-1] = unaligned_addr;
-  return aligned_addr;
-}
-
-static void __aligned_free(void *ptr) {
-  // retreive the stored unaligned address and use it to free the allocation
-  void* unaligned_addr = ((void**)ptr)[-1];
-  free(unaligned_addr);
-}
-
 int opae_sim::prepare_buffer(uint64_t len, void **buf_addr, uint64_t *wsid, int flags) {
   auto alloc = __aligned_malloc(CACHE_BLOCK_SIZE, len);
   if (alloc == NULL)
@@ -98,10 +105,10 @@ int opae_sim::prepare_buffer(uint64_t len, void **buf_addr, uint64_t *wsid, int
   buffer.data   = (uint64_t*)alloc;
   buffer.size   = len;
   buffer.ioaddr = uintptr_t(alloc); 
-  auto index = host_buffers_.size();
-  host_buffers_.emplace(index, buffer);
+  auto buffer_id = host_buffer_ids_++;
+  host_buffers_.emplace(buffer_id, buffer);
   *buf_addr = alloc;
-  *wsid = index;
+  *wsid = buffer_id;
   return 0;
 }
 
@@ -142,15 +149,9 @@ void opae_sim::write_mmio64(uint32_t mmio_num, uint64_t offset, uint64_t value)
   vortex_afu_->vcp2af_sRxPort_c0_mmioWrValid = 0;
 }
 
-void opae_sim::flush() {
-  // flush pending CCI requests  
-}
-
 ///////////////////////////////////////////////////////////////////////////////
 
 void opae_sim::reset() {  
-  
-  host_buffers_.clear();  
   cci_reads_.clear();
   cci_writes_.clear();
   vortex_afu_->vcp2af_sRxPort_c0_rspValid = 0;  
@@ -180,7 +181,6 @@ void opae_sim::reset() {
 }
 
 void opae_sim::step() {
-
   this->sRxPort_bus();
   this->sTxPort_bus();
   this->avs_bus();

driver/opae/vlsim/opae_sim.h

@@ -39,8 +39,6 @@ public:
 
   void read_mmio64(uint32_t mmio_num, uint64_t offset, uint64_t *value);
 
-  void flush();
-
 private: 
 
   typedef struct {
@@ -81,6 +79,7 @@ private:
   bool stop_;
 
   std::unordered_map<int64_t, host_buffer_t> host_buffers_;
+  int64_t host_buffer_ids_;
 
   std::list<mem_rd_req_t> mem_reads_ [PLATFORM_PARAM_LOCAL_MEMORY_BANKS];
 

driver/opae/vortex.cpp

@@ -314,11 +314,6 @@ extern void* vx_host_ptr(vx_buffer_h hbuffer) {
         return nullptr;
 
     vx_buffer_t* buffer = ((vx_buffer_t*)hbuffer);
-#ifdef USE_VLSIM
-    vx_device_t *device = ((vx_device_t*)buffer->hdevice);
-    fpgaFlush(device);
-#endif
-
     return buffer->host_ptr;
 }