text: /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
  /*
   * Copyright (c) 2011-2017 Los Alamos National Security, LLC. All rights
   *                         reserved.
   * Copyright (c) 2011      UT-Battelle, LLC. All rights reserved.
   * $COPYRIGHT$
   *
   * Additional copyrights may follow
   *
   * $HEADER$
   */
  
  #include "opal_config.h"
  
  #include "btl_ugni.h"
  #include "btl_ugni_frag.h"
  #include "btl_ugni_smsg.h"
  
  #include "opal/include/opal/align.h"
  
  
  static pthread_t mca_btl_ugni_progress_thread_id;
  
  static volatile int stop_progress_thread = 0;
  
  unsigned int mca_btl_ugni_progress_thread_wakeups = 0;
  
  static void *mca_btl_ugni_prog_thread_fn(void * data)
  {
      uint32_t which;
      gni_return_t status;
      gni_cq_handle_t cq_vec[1 + MCA_BTL_UGNI_MAX_DEV_HANDLES];
  
      struct mca_btl_ugni_module_t *btl = (mca_btl_ugni_module_t *)data;
      int cq_count = 1 + mca_btl_ugni_component.virtual_device_count;
  
      /*
       * need to block signals
       */
  
      cq_vec[0] = btl->smsg_remote_irq_cq;
      for (int i = 0 ; i < mca_btl_ugni_component.virtual_device_count ; ++i) {
          cq_vec[i + 1] = btl->devices[i].dev_rdma_local_irq_cq.gni_handle;
      }
  
      while (stop_progress_thread == 0) {
  
          /*
           * this ugni call doesn't need a lock
           */
  
          status = GNI_CqVectorMonitor(cq_vec,
                                       cq_count,
                                       -1,
                                       &which);
  
          if (status == GNI_RC_NOT_DONE) continue;
  
          if ((status == GNI_RC_SUCCESS) && (stop_progress_thread == 0)) {
              mca_btl_ugni_progress_thread_wakeups++;
              opal_progress();
          }
      }
  
      return (void *) (intptr_t) OPAL_SUCCESS;
  }
  
  int mca_btl_ugni_spawn_progress_thread(struct mca_btl_base_module_t *btl)
  {
      int rc, ret=OPAL_SUCCESS;
      pthread_attr_t attr;
  
      pthread_attr_init(&attr);
      rc = pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED);
      if (0 != rc) {
          BTL_ERROR(("btl/ugni pthread_attr_setdetachstate returned %s ",strerror(rc)));
          ret = OPAL_ERROR;
          goto fn_exit;
      }
  
      rc = pthread_create(&mca_btl_ugni_progress_thread_id,
                          &attr, mca_btl_ugni_prog_thread_fn, (void *)btl);
      if (0 != rc) {
          BTL_ERROR(("btl/ugni pthread_create returned %s ",strerror(rc)));
          ret = OPAL_ERROR;
          goto fn_exit;
      }
  
      rc = pthread_attr_destroy(&attr);
      if (0 != rc) {
          BTL_ERROR(("btl/ugni pthread_attr_destory returned %s ",strerror(rc)));
          ret = OPAL_ERROR;
      }
  
     fn_exit:
      return ret;
  }
  
  int mca_btl_ugni_kill_progress_thread(void)
  {
      int ret=OPAL_SUCCESS;
      void *thread_rc;
  
      stop_progress_thread = 1;
  
      /*
       * post a CQ to myself to wake my thread up
       */
  
      ret = mca_btl_ugni_post_cqwrite (mca_btl_ugni_component.modules[0].local_ep,
                                       &mca_btl_ugni_component.modules[0].devices[0].dev_rdma_local_cq,
                                       mca_btl_ugni_component.modules[0].devices[0].smsg_irq_mhndl,
                                       0xdead, NULL, NULL, NULL);
      /*
       * TODO: if error returned, need to kill off thread manually
       */
      if (OPAL_SUCCESS != ret) {
          /* force the thread to exit */
          pthread_cancel (mca_btl_ugni_progress_thread_id);
          goto fn_exit;
      }
  
      pthread_join (mca_btl_ugni_progress_thread_id, &thread_rc);
      if (0 != (intptr_t) thread_rc) {
          BTL_ERROR(("btl/ugni error returned from progress thread: %d", (int) (intptr_t) thread_rc));
          ret = (int)(intptr_t) thread_rc;
      }
  
     fn_exit:
      return ret;
  }
  
  