From 1b0bc8c6ce08eb934d05f068db0ee02a1a3497dd Mon Sep 17 00:00:00 2001
From: xin <xwang149@hawk.iit.edu>
Date: Tue, 22 Jan 2019 15:02:53 -0600
Subject: [PATCH 01/15] fix make error for buiding without conceputual

---
 src/workload/codes-conc-addon.c | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/workload/codes-conc-addon.c b/src/workload/codes-conc-addon.c
index 73592e9f..6b1d2ae8 100644
--- a/src/workload/codes-conc-addon.c
+++ b/src/workload/codes-conc-addon.c
@@ -8,7 +8,7 @@
 #include <codes/codes.h>
 #include <codes/codes-conc-addon.h>
 
-
+#ifdef USE_CONC
 /* list of available benchmarks begin */
 extern struct codes_conceptual_bench bisect_bench;
 extern struct codes_conceptual_bench cosmoflow_bench;
@@ -16,9 +16,11 @@ extern struct codes_conceptual_bench hotpotato_bench;
 extern struct codes_conceptual_bench latencyall_bench;
 extern struct codes_conceptual_bench latency_bench;
 /* list of available benchmarks end */
+#endif
 
 static struct codes_conceptual_bench const * bench_array_default[] =
 {
+#ifdef USE_CONC
     /* default benchmarks begin */
     &bisect_bench,
     &cosmoflow_bench,
@@ -26,6 +28,7 @@ static struct codes_conceptual_bench const * bench_array_default[] =
     &latencyall_bench,
     &latency_bench,
     /* default benchmarks end */
+#endif
     NULL
 };
 

From 2bafea6dddf8b7d14c21d13c4f28224ac0131919 Mon Sep 17 00:00:00 2001
From: Xin Wang <xwang149@hawk.iit.edu>
Date: Tue, 22 Jan 2019 15:09:29 -0600
Subject: [PATCH 02/15] Delete bisect.c

---
 scripts/conceptual_benchmarks/bisect.c | 1277 ------------------------
 1 file changed, 1277 deletions(-)
 delete mode 100644 scripts/conceptual_benchmarks/bisect.c

diff --git a/scripts/conceptual_benchmarks/bisect.c b/scripts/conceptual_benchmarks/bisect.c
deleted file mode 100644
index 26595494..00000000
--- a/scripts/conceptual_benchmarks/bisect.c
+++ /dev/null
@@ -1,1277 +0,0 @@
-/**********************************************************************
- * This file was generated by coNCePTuaL on Mon Jan  7 23:10:02 2019
- * using the c_mpi backend (C + MPI).
- * Do not modify this file; modify /Users/xin/macworkspace/codes-dev/codes/scripts/conceptual_benchmarks/bisect.ncptl instead.
- *
- * Entire source program
- * ---------------------
- *   ###################################################
- *   # Measure random bisection-bandwidth patterns     #
- *   # By Scott Pakin <pakin@lanl.gov>                 #
- *   #                                                 #
- *   # Inspired by Hoefler, Schneider, and Lumsdaine's #
- *   # "Multistage Switches are not Crossbars" paper   #
- *   ###################################################
- *   
- *   Require language version "1.5".
- *   
- *   nummsgs is "Number of messages per trial" and comes from "--nummsgs" or "-n" with default 100.
- *   wups is "Number of warmup messages" and comes from "--wups" or "-w" with default 3.
- *   msgsize is "Message size in bytes" and comes from "--bytes" or "-b" with default 1M.
- *   numtrials is "Number of bisection patterns" and comes from "--trials" or "-t" with default 5000.
- *   rounding is "Round measurements to the nearest N" and comes from "--round" or "-r" with default 5.
- *   
- *   For each trial in {1, ..., numtrials} {
- *     task 0 is assigned to processor 0 then
- *     task 0 outputs "Testing random bisection pattern " and trial and "/" and numtrials then
- *     all tasks are assigned to a random processor then
- *     tasks src such that src is even send wups msgsize-byte messages to task src+1 then
- *     all tasks synchronize then
- *     all tasks reset their counters then
- *     tasks src such that src is even send nummsgs msgsize-byte messages to task src+1 then
- *     all tasks synchronize then
- *     all tasks log a histogram of rounding*round(total_bytes*1E6/(elapsed_usecs*1M)/rounding) as "Bandwidth (MiB/s)"
- *   }
- **********************************************************************/
-
-/*****************
- * Include files *
- *****************/
-
-/* Header files needed by all C-based backends */
-#include <stdio.h>
-#include <string.h>
-#include <ncptl/ncptl.h>
-
-/* Header files specific to the c_mpi backend */
-#include <mpi.h>
-#include <stdarg.h>
-
-/**********
- * Macros *
- **********/
-
-/* Define the maximum loop trip count that we're willing to unroll fully. */
-#define CONC_MAX_UNROLL 5
-
-/* Specify the minimum number of trial iterations in each FOR <time> loop. */
-#define CONC_FOR_TIME_TRIALS 1
-
-/* Define a macro that rounds a double to a ncptl_int. */
-#define CONC_DBL2INT(D) ((ncptl_int)((D)+0.5))
-
-/* Define a macro that increments a buffer pointer by a byte offset. */
-#define CONC_GETBUFPTR(S) ((void *)((char *)thisev->s.S.buffer + thisev->s.S.bufferofs))
-
-/* Implement ncptl_func_task_of in terms of ncptl_physical_to_virtual. */
-#define ncptl_func_task_of(P) ((ncptl_int)(P) < 0LL || (ncptl_int)(P) >= var_num_tasks ? -1LL : ncptl_physical_to_virtual (procmap, (ncptl_int)(P)))
-#define ncptl_dfunc_task_of(P) ((double) ncptl_func_task_of(P))
-
-/* Implement ncptl_func_processor_of in terms of ncptl_virtual_to_physical. */
-#define ncptl_func_processor_of(V) ((ncptl_int)(V) < 0LL || (ncptl_int)(V) >= var_num_tasks ? -1LL : ncptl_virtual_to_physical (procmap, (ncptl_int)(V)))
-#define ncptl_dfunc_processor_of(V) ((double) ncptl_func_processor_of(V))
-
-/* Define a macro that increments REDUCE's alternate buffer pointer by a byte offset. */
-#define CONC_GETALTBUFPTR(S) ((void *)((char *)thisev->s.S.altbuffer + thisev->s.S.bufferofs))
-
-/* Estimate the number of unique communicators that this program will need.
- * (The tradeoff is one of initialization time versus memory consumption.) */
-#define ESTIMATED_COMMUNICATORS 128
-
-/* Specify an operation to use for all reduction operations. */
-#define REDUCE_OPERATION MPI_SUM
-#define REDUCE_OPERATION_NAME "MPI_SUM"
-
-
-/*********************
- * Type declarations *
- *********************/
-
-/* Enumerate the various mechanisms used to implement MULTICAST statements. */
-typedef enum {
-CONC_MCAST_MPI_BCAST,       /* One to many */
-CONC_MCAST_MPI_ALLTOALL,    /* Many to many, same data to all */ 
-CONC_MCAST_MPI_ALLTOALLV,   /* General many to many */
-CONC_MCAST_MPI_NUM_FUNCS    /* Number of the above */
-} CONC_MCAST_MPI_FUNC;
-
-/* Define the type of event to perform. */
-typedef enum {
-EV_SEND,     /* Synchronous send */
-EV_ASEND,    /* Asynchronous send */
-EV_RECV,     /* Synchronous receive */
-EV_ARECV,    /* Asynchronous receive */
-EV_WAIT,     /* Wait for all asynchronous sends/receives to complete */
-EV_DELAY,    /* Spin or sleep */
-EV_TOUCH,    /* Touch a region of memory */
-EV_SYNC,     /* Barrier synchronization */
-EV_RESET,    /* Reset counters */
-EV_STORE,    /* Store all counters' current values */
-EV_RESTORE,  /* Restore the previously pushed counter values */
-EV_FLUSH,    /* Compute aggregate functions for log-file columns */
-EV_MCAST,    /* Synchronous multicast */
-EV_REDUCE,   /* Reduction with or without a subsequent multicast */
-EV_BTIME,    /* Beginning of a timed loop */
-EV_ETIME,    /* Ending of a timed loop */
-EV_REPEAT,   /* Repeatedly process the next N events */
-EV_SUPPRESS, /* Suppress writing to the log and standard output */
-EV_NEWSTMT,  /* Beginning of a new top-level statement */
-EV_CODE,     /* None of the above */
-NUM_EVS      /* Number of event types in CONC_EVENT_TYPE */
-} CONC_EVENT_TYPE;
-
-/* Describe a synchronous or asynchronous send event. */
-typedef struct {
-ncptl_int dest;         /* Destination task */
-ncptl_int size;         /* Number of bytes to send */
-ncptl_int alignment;    /* Message alignment (in bytes) */
-ncptl_int pendingsends; /* # of outstanding sends */
-ncptl_int pendingrecvs; /* # of outstanding receives */
-ncptl_int buffernum;    /* Buffer # to send from */
-ncptl_int bufferofs;    /* Byte offset into the message buffer */
-ncptl_int tag;          /* Tag to use for selective receives */
-int misaligned;         /* 1=misaligned from a page; 0=align as specified */
-int touching;           /* 1=touch every word before sending */
-int verification;       /* 1=fill message buffer with known contents */
-void *buffer;           /* Pointer to message memory */
-MPI_Request * handle;   /* MPI handle representing an asynchronous send */
-} CONC_SEND_EVENT;
-
-/* Describe a synchronous or asynchronous receive event. */
-typedef struct {
-ncptl_int source;       /* Source task */
-ncptl_int size;         /* Number of bytes to receive */
-ncptl_int alignment;    /* Message alignment (in bytes) */
-ncptl_int pendingsends; /* # of outstanding sends */
-ncptl_int pendingrecvs; /* # of outstanding receives */
-ncptl_int buffernum;    /* Buffer # to receive into */
-ncptl_int bufferofs;    /* Byte offset into the message buffer */
-ncptl_int tag;          /* Tag to use for selective receives */
-int misaligned;         /* 1=misaligned from a page; 0=align as specified */
-int touching;           /* 1=touch every word after reception */
-int verification;       /* 1=verify that all bits are correct */
-void *buffer;           /* Pointer to message memory */
-MPI_Request * handle;   /* MPI handle representing an asynchronous receive */
-} CONC_RECV_EVENT;
-
-/* Describe a wait-for-asynchronous-completions event. */
-typedef struct {
-ncptl_int numsends;     /* # of sends we expect to complete. */
-ncptl_int numrecvs;     /* # of receives we expect to complete. */
-ncptl_int numrecvbytes; /* # of bytes we expect to receive-complete */
-ncptl_int *touchedlist;    /* List of receives that need to be touched */
-ncptl_int numtouches;         /* # of elements in the above */
-} CONC_WAIT_EVENT;
-
-/* Describe a spin or sleep delay. */
-typedef struct {
-uint64_t microseconds;  /* Length of delay in microseconds */
-int spin0sleep1;        /* 0=spin; 1=sleep */
-} CONC_DELAY_EVENT;
-
-/* Describe a barrier synchronization event. */
-typedef struct {
-MPI_Comm communicator;   /* Set of tasks to synchronize */
-} CONC_SYNC_EVENT;
-
-/* Describe a walk over a memory-region. */
-typedef struct {
-ncptl_int regionbytes;  /* Size in bytes of the region to touch */
-ncptl_int bytestride;   /* Stride in bytes to touch */
-ncptl_int numaccesses;  /* Number of words to touch */
-ncptl_int wordsize;     /* Size in bytes of each touch */
-ncptl_int firstbyte;    /* Byte offset of the first byte to touch */
-} CONC_TOUCH_EVENT;
-
-/* Describe a synchronous multicast event. */
-typedef struct {
-ncptl_int source;       /* Source task, -1 in the many-to-many case */
-ncptl_int size;         /* Number of bytes to send */
-ncptl_int alignment;    /* Message alignment (in bytes) */
-ncptl_int pendingsends; /* # of outstanding sends */
-ncptl_int pendingrecvs; /* # of outstanding receives */
-ncptl_int buffernum;    /* Buffer # to send/receive from */
-ncptl_int bufferofs;    /* Byte offset into the message buffer */
-ncptl_int tag;          /* Tag to use for selective receives */
-int misaligned;         /* 1=misaligned from a page; 0=align as specified */
-int touching;           /* 1=touch every word before sending */
-int verification;       /* 1=verify that all bits are correct */
-void *buffer;           /* Pointer to message memory */
-ncptl_int size2;   /* Number of bytes to receive in the many-to-many case */
-ncptl_int bufferofs2;   /* Byte offset into the message buffer in the many-to-many case */
-void * buffer2;   /* Pointer to receive-message memory in the many-to-many case */
-MPI_Comm communicator;   /* Set of tasks to multicast to/from */
-int root;   /* source's rank within communicator */
-int * sndvol;   /* Volume of data to send to each rank in the communicator */
-int * snddisp;   /* Offset from buffer of each message to send */
-int * rcvvol;   /* Volume of data to receive from each rank in the communicator */
-int * rcvdisp;   /* Offset from buffer2 of each message to receive */
-CONC_MCAST_MPI_FUNC mpi_func;   /* MPI function to use to perform the multicast */
-} CONC_MCAST_EVENT;
-
-/* Describe a reduction event. */
-typedef struct {
-ncptl_int numitems;     /* # of items to reduce */
-ncptl_int itemsize;     /* # of bytes per item */
-ncptl_int alignment;    /* Message alignment (in bytes) */
-ncptl_int pendingsends; /* # of outstanding sends */
-ncptl_int pendingrecvs; /* # of outstanding receives */
-ncptl_int buffernum;    /* Buffer # to send/receive from */
-ncptl_int bufferofs;    /* Byte offset into the message buffer */
-ncptl_int tag;          /* Tag to use for selective receives */
-int misaligned;         /* 1=misaligned from a page; 0=align as specified */
-int touching;           /* 1=touch every word before sending/after receiving */
-int sending;            /* 1=we're a sender */
-int receiving;          /* 1=we're a receiver */
-void *buffer;           /* Pointer to message memory */
-void * altbuffer;   /* Pointer to additional message memory */
-MPI_Comm sendcomm;   /* Set of tasks to reduce from */
-MPI_Comm recvcomm;   /* Set of tasks to reduce to */
-MPI_Datatype datatype;   /* MPI datatype to reduce */
-int reducetype;   /* 0=reduce; 1=allreduce; 2=reduce+bcast */
-ncptl_int reduceroot;   /* Root task of the reduction if reducetype is 0 or 2 */
-ncptl_int bcastroot;   /* Root task of the multicast if reducetype is 2 */
-} CONC_REDUCE_EVENT;
-
-/* Describe an event representing the beginning of a timed loop. */
-typedef struct {
-uint64_t usecs;         /* Requested loop duration */
-uint64_t warmup_usecs;  /* Requested duration of warmup loops */
-uint64_t starttime;     /* Time at which the loop state last changed */
-uint64_t itersleft;     /* # of iterations remaining */
-uint64_t previters;     /* # of iterations we performed last time */
-int prev_quiet;         /* Previous value of suppress_output */
-int timing_trial;       /* 1=performing a timing trial; 0=running for real */
-volatile int finished;  /* 1=time has expired; 0=still ticking */
-ncptl_int var_total_bytes;   /* Cached copy of var_total_bytes to restore after the trial runs */
-} CONC_BTIME_EVENT;
-
-/* Describe an event representing the end of a timed loop. */
-typedef struct {
-ncptl_int begin_event;  /* Index into eventlist[] of the corresponding BTIME event */
-} CONC_ETIME_EVENT;
-
-/* Describe an event representing repetitions of subsequent events. */
-typedef struct {
-ncptl_int end_event;    /* Index into eventlist[] of the last event to repeat */
-ncptl_int numreps;      /* # of repetitions to perform */
-} CONC_REPEAT_EVENT;
-
-/* Describe an event representing output suppression (either on or off). */
-typedef struct conc_suppress_event {
-int quiet;              /* 0=allow output; 1=suppress it */
-int prev_quiet;         /* Previous value of suppress_output */
-ncptl_int matching_event;  /* Event ID of the "suppression on" event */
-ncptl_int var_total_bytes;   /* Cached copy of var_total_bytes to restore after re-enabling output */
-uint64_t stop_elapsed_usecs;   /* Time at which we suppressed output */
-} CONC_SUPPRESS_EVENT;
-
-/* Describe an event representing arbitrary code to execute at run time. */
-typedef struct {
-ncptl_int number;       /* Unique number corresponding to a specific piece of code */NCPTL_VIRT_PHYS_MAP *procmap;  /* Current mapping between tasks and processors */
-ncptl_int var_numtrials;   /* Copy of var_numtrials to use within a piece of code */
-ncptl_int var_trial;   /* Copy of var_trial to use within a piece of code */
-ncptl_int var_rounding;   /* Copy of var_rounding to use within a piece of code */
-} CONC_CODE_EVENT;
-
-/* Describe an arbitrary coNCePTuaL event. */
-typedef struct {
-CONC_EVENT_TYPE type;          /* Type of event */
-union {
-CONC_SEND_EVENT send;          /* Send state */
-CONC_RECV_EVENT recv;          /* Receive state */
-CONC_WAIT_EVENT wait;          /* Wait-for-completions state */
-CONC_DELAY_EVENT delay;        /* State for spins and sleeps */
-CONC_TOUCH_EVENT touch;        /* State for memory touching */
-CONC_SYNC_EVENT sync;          /* Synchronization state */
-CONC_MCAST_EVENT mcast;        /* Multicast state */
-CONC_REDUCE_EVENT reduce;      /* Reduction state */
-CONC_BTIME_EVENT btime;        /* Timed-loop state */
-CONC_ETIME_EVENT etime;        /* Additional timed-loop state */
-CONC_REPEAT_EVENT rep;         /* Repeated-events state */
-CONC_SUPPRESS_EVENT suppress;  /* State for suppressing output */
-CONC_CODE_EVENT code;          /* State for arbitrary code */
-} s;
-} CONC_EVENT;
-
-/* Fully specify an arbitrary for() loop (used by FOR EACH). */
-typedef struct {
-NCPTL_QUEUE *list_comp;  /* NULL=ordinary list; other=list comprehension values */
-int integral;        /* 1=integral values; 0=floating-point values */
-enum {               /* Comparison of loop variable to end variable */
-CONC_LEQ,                /* Increasing progression */
-CONC_GEQ                 /* Decreasing progression */
-} comparator;
-enum {               /* How to increment the loop variable */
-CONC_ADD,                /* Arithmetically */
-CONC_MULT,               /* Geometrically increasing */
-CONC_DIV                 /* Geometrically decreasing */
-} increment;
-union {
-struct {
-ncptl_int loopvar;   /* Loop variable */
-ncptl_int prev_loopvar; /* Previous value of loop variable */
-ncptl_int startval;  /* Initial value of loop variable */
-ncptl_int endval;    /* Value not to exceed */
-ncptl_int incval;    /* Loop-variable increment */
-} i;
-struct {
-double loopvar;      /* Loop variable */
-double prev_loopvar; /* Previous value of loop variable */
-double startval;     /* Initial value of loop variable */
-double endval;       /* Value not to exceed */
-double incval;       /* Loop-variable increment */
-} d;
-} u;
-} LOOPBOUNDS;
-
-/********************
- * Global variables *
- ********************/
-
-/* Variables exported to coNCePTuaL programs */
-static ncptl_int var_bytes_received = 0;   /* Total number of bytes received */
-static ncptl_int var_msgs_received = 0;   /* Total number of messages received */
-static ncptl_int var_bit_errors = 0;   /* Total number of bit errors observed */
-static ncptl_int var_total_msgs = 0;   /* Sum of messages sent and messages received */
-static ncptl_int var_msgs_sent = 0;   /* Total number of messages sent */
-static ncptl_int var_bytes_sent = 0;   /* Total number of bytes sent */
-static ncptl_int var_num_tasks = 1;   /* Number of tasks running the program */
-static ncptl_int var_elapsed_usecs = 0;   /* Elapsed time in microseconds */
-static ncptl_int var_total_bytes = 0;   /* Sum of bytes sent and bytes received */
-
-/* Dummy variable to help mark other variables as used */
-union {
-ncptl_int ni;
-int i;
-void *vp;
-} conc_dummy_var;
-
-/* Variables used internally by boilerplate code */
-static uint64_t starttime;   /* Time the clock was last reset (microseconds) */
-static ncptl_int pendingrecvs = 0;   /* Current # of outstanding receives */
-static ncptl_int pendingrecvbytes = 0; /* Current # of bytes in outstanding receives */
-static NCPTL_QUEUE *touchedqueue;      /* Queue of asynchronous receives to touch */
-static ncptl_int pendingsends = 0;   /* Current # of outstanding sends */
-static NCPTL_QUEUE *eventqueue;   /* List of coNCePTuaL events to perform */
-static int within_time_loop = 0;   /* 1=we're within a FOR <time> loop */
-static int suppress_output = 0;    /* 1=suppress output to stdout and the log file */
-static void *touch_region = NULL;   /* Memory region to touch */
-static ncptl_int touch_region_size = 0;   /* # of bytes in the above */
-static int virtrank;    /* This task's virtual rank in the computation */
-static int physrank;    /* This task's physical rank in the computation */
-static NCPTL_VIRT_PHYS_MAP *procmap;  /* Virtual to physical rank mapping */
-static ncptl_int random_seed;   /* Seed for the random-number generator */
-static NCPTL_LOG_FILE_STATE *logstate;   /* Opaque object representing all log-file state */
-static char *logfile_uuid;   /* Execution UUID to write to every log file */
-static char *logfiletmpl;   /* Template for the log file's name */
-static char *logfiletmpl_default;   /* Default value of the above */
-
-/* Global variables specific to the c_mpi backend */
-static ncptl_int mpi_is_running = 0;   /* 1=MPI has been initialized */
-static NCPTL_QUEUE * recvreqQ;   /* List of MPI receive requests */
-static MPI_Request * recvrequests;   /* List version of recvreqQ */
-static NCPTL_QUEUE * recvstatQ;   /* List of MPI receive statuses */
-static MPI_Status * recvstatuses;   /* List version of recvstatQ */
-static NCPTL_QUEUE * sendreqQ;   /* List of MPI send requests */
-static MPI_Request * sendrequests;   /* List version of sendreqQ */
-static NCPTL_QUEUE * sendstatQ;   /* List of MPI send statuses */
-static MPI_Status * sendstatuses;   /* List version of sendstatQ */
-static NCPTL_SET * communicators;   /* Map from an array of processor flags to an MPI communicator */
-static MPI_Errhandler mpi_error_handler;   /* Handle to handle_MPI_error() */
-static ncptl_int mpi_tag_ub;   /* Upper bound on an MPI tag value */
-static ncptl_int conc_mcast_tallies[CONC_MCAST_MPI_NUM_FUNCS] = {0};   /* Tallies of (static) multicast implementation functions */
-
-/* Program-specific variables */
-ncptl_int var_nummsgs;   /* Number of messages per trial (command-line argument) */
-ncptl_int var_wups;   /* Number of warmup messages (command-line argument) */
-ncptl_int var_msgsize;   /* Message size in bytes (command-line argument) */
-ncptl_int var_numtrials;   /* Number of bisection patterns (command-line argument) */
-ncptl_int var_rounding;   /* Round measurements to the nearest N (command-line argument) */
-
-/*************************
- * Function declarations *
- *************************/
-
-/* Make MPI errors invoke ncptl_fatal(). */
-static void handle_MPI_error (MPI_Comm *comm, int *errcode, ...)
-{
-va_list args;
-char errstring[MPI_MAX_ERROR_STRING];
-int errstrlen;
-
-va_start (args, errcode);
-if (MPI_Error_string (*errcode, errstring, &errstrlen) == MPI_SUCCESS)
-ncptl_fatal ("MPI run-time error: %s", errstring);
-else
-ncptl_fatal ("MPI aborted with unrecognized error code %d", *errcode);
-conc_dummy_var.vp = (void *) comm;   /* Prevent the compiler from complaining that comm is unused. */
-va_end (args);
-}
-
-/* Perform the equivalent of MPI_Comm_rank() for an arbitrary process. */
-static int rank_in_MPI_communicator (MPI_Comm subcomm, int global_rank)
-{
-  MPI_Group world_group;   /* Group associated with MPI_COMM_WORLD */
-  MPI_Group subgroup;      /* Group associate with subcomm */
-  int subrank;             /* global_rank's rank within subcomm */
-
-  MPI_Comm_group (MPI_COMM_WORLD, &world_group);
-  MPI_Comm_group (subcomm, &subgroup);
-  MPI_Group_translate_ranks (world_group, 1, &global_rank, subgroup, &subrank);
-  return subrank;
-}
-
-/* Map an arbitrary tag to within MPI's valid range of [0, mpi_tag_ub]. */
-static ncptl_int map_tag_into_MPI_range (ncptl_int tag)
-{
-if (tag == NCPTL_INT_MIN)
- /* Avoid taking the absolute value of NCPTL_INT_MIN. */
-tag = 555666773LL;   /* Arbitrary value */
-tag = ncptl_func_abs (tag);   /* Only nonnegatives values are allowed. */
-if (mpi_tag_ub < NCPTL_INT_MAX)
-tag %= mpi_tag_ub + 1;
-return tag;
-}
-
-/* Given an array of task in/out booleans return an MPI
- * communicator that represents the "in" tasks. */
-static MPI_Comm define_MPI_communicator (char *procflags)
-{
-MPI_Comm *existing_comm;    /* Previously defined MPI communicator */
-MPI_Comm new_comm;          /* Newly defined MPI communicator */
-
-existing_comm = (MPI_Comm *) ncptl_set_find (communicators, (void *)procflags);
-if (existing_comm)
-return *existing_comm;
-(void) MPI_Comm_split (MPI_COMM_WORLD, (int)procflags[physrank], physrank, &new_comm);
-(void) MPI_Errhandler_set (new_comm, mpi_error_handler);
-ncptl_set_insert (communicators, (void *)procflags, (void *)&new_comm);
-return define_MPI_communicator (procflags);
-}
-
-/* Return 1 if a sequence loop will take at least one trip. */
-static int conc_seq_nonempty (LOOPBOUNDS *seq)
-{
-ncptl_int startval;   /* Integer version of seq's startval element */
-ncptl_int endval;   /* Integer version of seq's endval element */
-
-if (seq->integral) {
-startval = seq->u.i.startval;
-endval = seq->u.i.endval;
-}
-else {
-startval = CONC_DBL2INT (seq->u.d.startval);
-endval = CONC_DBL2INT (seq->u.d.endval);
-}
-switch (seq->comparator) {
-case CONC_LEQ:
-return startval <= endval;
-
-case CONC_GEQ:
-return startval >= endval;
-
-default:
-ncptl_fatal ("Internal error -- unknown comparator");
-}
-return -1;     /* Appease idiotic compilers. */
-}
-
-/* Initialize a sequence loop. */
-static void conc_seq_init (LOOPBOUNDS *seq)
-{
-if (seq->integral) {
-seq->u.i.loopvar = seq->u.i.startval;
-seq->u.i.prev_loopvar = seq->u.i.loopvar - 1;
-}
-else {
-seq->u.d.loopvar = seq->u.d.startval;
-seq->u.d.prev_loopvar = seq->u.d.loopvar - 1.0;
-}
-}
-
-/* Return 1 if a sequence loop should continue, 0 when finished. */
-static int conc_seq_continue (LOOPBOUNDS *seq)
-{
-LOOPBOUNDS seq_int;   /* Integer equivalent of *seq */
-if (seq->integral)
-seq_int = *seq;
-else {
-seq_int.u.i.loopvar = CONC_DBL2INT (seq->u.d.loopvar);
-seq_int.u.i.prev_loopvar = CONC_DBL2INT (seq->u.d.prev_loopvar);
-seq_int.u.i.endval = CONC_DBL2INT (seq->u.d.endval);
-}
-
-if (seq_int.u.i.loopvar == seq_int.u.i.prev_loopvar)
-return 0;
-switch (seq->comparator) {
-case CONC_LEQ:
-return seq_int.u.i.loopvar <= seq_int.u.i.endval;
-
-case CONC_GEQ:
-return seq_int.u.i.loopvar >= seq_int.u.i.endval;
-
-default:
-ncptl_fatal ("Internal error -- unknown comparator");
-}
-return -1;     /* Appease idiotic compilers. */
-}
-
-/* Proceed to the next iteration of a sequence loop. */
-static void conc_seq_next (LOOPBOUNDS *seq)
-{
-if (seq->integral) {
-seq->u.i.prev_loopvar = seq->u.i.loopvar;
-switch (seq->increment) {
-case CONC_ADD:
-seq->u.i.loopvar += seq->u.i.incval;
-break;
-
-case CONC_MULT:
-seq->u.i.loopvar *= seq->u.i.incval;
-break;
-
-case CONC_DIV:
-seq->u.i.loopvar /= seq->u.i.incval;
-break;
-
-default:
-ncptl_fatal ("Internal error -- unknown incrementer");
-}
-}
-else {
-seq->u.d.prev_loopvar = seq->u.d.loopvar;
-switch (seq->increment) {
-case CONC_ADD:
-seq->u.d.loopvar += seq->u.d.incval;
-break;
-
-case CONC_MULT:
-seq->u.d.loopvar *= seq->u.d.incval;
-break;
-
-case CONC_DIV:
-seq->u.d.loopvar /= seq->u.d.incval;
-break;
-
-default:
-ncptl_fatal ("Internal error -- unknown incrementer");
-}
-}
-}
-
-/* Inhibit the compiler from complaining that
- * certain variables are defined but not used.
- * This function should never be called. */
-void conc_mark_variables_used (void)
-{
-conc_dummy_var.ni = var_bytes_received;
-conc_dummy_var.ni = var_msgs_received;
-conc_dummy_var.ni = var_bit_errors;
-conc_dummy_var.ni = var_total_msgs;
-conc_dummy_var.ni = var_msgs_sent;
-conc_dummy_var.ni = var_bytes_sent;
-conc_dummy_var.ni = var_num_tasks;
-conc_dummy_var.ni = var_elapsed_usecs;
-conc_dummy_var.ni = var_total_bytes;
-conc_dummy_var.ni = pendingrecvbytes;
-conc_dummy_var.ni = touch_region_size;
-conc_dummy_var.vp = touch_region;
-conc_dummy_var.i  = within_time_loop;
-conc_dummy_var.i  = suppress_output;
-rank_in_MPI_communicator (MPI_COMM_WORLD, 0);
-}
-
-/* Allocate a new event of a given type and return a pointer to it. */
-static CONC_EVENT *conc_allocate_event (CONC_EVENT_TYPE type)
-{
-CONC_EVENT *newevent = (CONC_EVENT *) ncptl_queue_allocate (eventqueue);
-
-newevent->type = type;
-return newevent;
-}
-
-/* Declare an exit handler that gets called automatically when the
- * program terminates, whether successfully or not. */
-static void conc_exit_handler (void)
-{
-if (mpi_is_running)
-MPI_Abort (MPI_COMM_WORLD, 1);
-}
-
-
-/* Initialize coNCePTuaL, the messaging layer, and this program itself. */
-static void conc_initialize (int argc, char *argv[])
-{
- /* Variables needed by all C-based backends */
-CONC_EVENT * eventlist;   /* List of events to execute */
-ncptl_int numevents;   /* Number of entries in eventlist[] */
-int help_only = 0;   /* 1=User specified --help; save time by skipping ncptl_init() */
-char * argv0 = strrchr(argv[0], '/') ? strrchr(argv[0], '/')+1 : argv[0];   /* Base name of the executable program */
-int i;   /* Generic loop variable */
-
- /* Declare all of our command-line arguments. */
-NCPTL_CMDLINE arguments[] = {
-{ NCPTL_TYPE_INT, NULL, "seed", 'S', "Seed for the random-number generator", {0}},
-{ NCPTL_TYPE_STRING, NULL, "logfile", 'L', "Log-file template", {0}},
-{ NCPTL_TYPE_INT, NULL, "nummsgs", 'n', "Number of messages per trial", {0}},
-{ NCPTL_TYPE_INT, NULL, "wups", 'w', "Number of warmup messages", {0}},
-{ NCPTL_TYPE_INT, NULL, "bytes", 'b', "Message size in bytes", {0}},
-{ NCPTL_TYPE_INT, NULL, "trials", 't', "Number of bisection patterns", {0}},
-{ NCPTL_TYPE_INT, NULL, "round", 'r', "Round measurements to the nearest N", {0}}
-};
-
- /* Incorporate the complete coNCePTuaL source code as an array
-  * for use by ncptl_log_write_prologue(). */
-char *sourcecode[] = {
-"###################################################",
-"# Measure random bisection-bandwidth patterns     #",
-"# By Scott Pakin <pakin@lanl.gov>                 #",
-"#                                                 #",
-"# Inspired by Hoefler, Schneider, and Lumsdaine's #",
-"# \"Multistage Switches are not Crossbars\" paper   #",
-"###################################################",
-"",
-"Require language version \"1.5\".",
-"",
-"nummsgs is \"Number of messages per trial\" and comes from \"--nummsgs\" or \"-n\" with default 100.",
-"wups is \"Number of warmup messages\" and comes from \"--wups\" or \"-w\" with default 3.",
-"msgsize is \"Message size in bytes\" and comes from \"--bytes\" or \"-b\" with default 1M.",
-"numtrials is \"Number of bisection patterns\" and comes from \"--trials\" or \"-t\" with default 5000.",
-"rounding is \"Round measurements to the nearest N\" and comes from \"--round\" or \"-r\" with default 5.",
-"",
-"For each trial in {1, ..., numtrials} {",
-"  task 0 is assigned to processor 0 then",
-"  task 0 outputs \"Testing random bisection pattern \" and trial and \"/\" and numtrials then",
-"  all tasks are assigned to a random processor then",
-"  tasks src such that src is even send wups msgsize-byte messages to task src+1 then",
-"  all tasks synchronize then",
-"  all tasks reset their counters then",
-"  tasks src such that src is even send nummsgs msgsize-byte messages to task src+1 then",
-"  all tasks synchronize then",
-"  all tasks log a histogram of rounding*round(total_bytes*1E6/(elapsed_usecs*1M)/rounding) as \"Bandwidth (MiB/s)\"",
-"}",
-NULL
-};
-
- /* Variables specific to the c_mpi backend */
-int num_tasks;   /* int version of var_num_tasks needed by MPI_Comm_size() */
-char * procflags;   /* Array of 1s representing an all-task MPI communicator */
-MPI_Comm comm_world = MPI_COMM_WORLD;   /* Copy of MPI_COMM_WORLD that we can take the address of */
-void * attr_val;   /* Pointed to the value of MPI_TAG_UB */
-int attr_flag = 0;   /* true=MPI_TAG_UB was extracted; false=not extracted */
-char log_key_str[128];   /* String representing the range of valid MPI tags */
-
- /* As a special case, if the command line contains --help, then skip
-  * the coNCePTuaL initialization step. */
-for (i=1; i<argc; i++)
-if (!strcmp(argv[i], "--"))
-break;
-else
-if (!strcmp(argv[i], "--help") || !strcmp(argv[i], "-?")) {
-argv[1] = "-?";   /* Guaranteed to work, even with getopt() */
-help_only = 1;
-break;
-}
-
- /* Perform various initializations specific to the c_mpi backend. */
- /* Initialize MPI. */
-(void) MPI_Init(&argc, &argv);
-mpi_is_running = 1;
-
- /* Initialize the coNCePTuaL run-time library. */
-if (!help_only)
-ncptl_init (NCPTL_RUN_TIME_VERSION, argv[0]);
-(void) atexit (conc_exit_handler);
-
- /* Initialize the communication routines needed by the c_mpi backend. */
-(void) MPI_Errhandler_create ((MPI_Handler_function *)handle_MPI_error, &mpi_error_handler);
-(void) MPI_Errhandler_set (MPI_COMM_WORLD, mpi_error_handler);
-(void) MPI_Comm_rank(MPI_COMM_WORLD, &physrank);
-(void) MPI_Comm_size(MPI_COMM_WORLD, &num_tasks);
-var_num_tasks = (ncptl_int) num_tasks;
-(void) MPI_Comm_get_attr(MPI_COMM_WORLD, MPI_TAG_UB, &attr_val, &attr_flag);
-mpi_tag_ub = (ncptl_int) (attr_flag ? *(int *)attr_val : 32767);
-
- /* Seed the random-task-number generator. */
-random_seed = ncptl_seed_random_task(random_seed, 0);
-arguments[0].defaultvalue.intval = random_seed;
-
- /* Generate and broadcast a UUID. */
-logfile_uuid = ncptl_log_generate_uuid();
-(void) MPI_Bcast ((void *)logfile_uuid, 37, MPI_CHAR, 0, MPI_COMM_WORLD);
-
- /* Plug variables and default values into the NCPTL_CMDLINE structure. */
-arguments[0].variable = (CMDLINE_VALUE *) &random_seed;
-arguments[0].defaultvalue.intval = random_seed;
-arguments[1].variable = (CMDLINE_VALUE *) &logfiletmpl;
-arguments[2].variable = (CMDLINE_VALUE *) &var_nummsgs;
-arguments[2].defaultvalue.intval = 100LL;
-arguments[3].variable = (CMDLINE_VALUE *) &var_wups;
-arguments[3].defaultvalue.intval = 3LL;
-arguments[4].variable = (CMDLINE_VALUE *) &var_msgsize;
-arguments[4].defaultvalue.intval = 1048576LL;
-arguments[5].variable = (CMDLINE_VALUE *) &var_numtrials;
-arguments[5].defaultvalue.intval = 5000LL;
-arguments[6].variable = (CMDLINE_VALUE *) &var_rounding;
-arguments[6].defaultvalue.intval = 5LL;
-logfiletmpl_default = (char *) ncptl_malloc (strlen(argv0) + 15, 0);
-sprintf (logfiletmpl_default, "%s-%%p.log", argv0);
-arguments[1].defaultvalue.stringval = logfiletmpl_default;
-
- /* Parse the command line. */
-mpi_is_running = 0;   /* Don't invoke MPI_Abort() after --help. */
-ncptl_parse_command_line (argc, argv, arguments, sizeof(arguments)/sizeof(NCPTL_CMDLINE));
-if (help_only)
-ncptl_fatal ("Internal error in the c_generic backend: failed to exit after giving help");
-mpi_is_running = 1;
-
- /* Broadcast the random-task-number generator's seed. */
-{
-int rndseed_int = (int) random_seed;   /* Version of random_seed with int type */
-(void) MPI_Bcast ((void *)&rndseed_int, 1, MPI_INT, 0, MPI_COMM_WORLD);
-random_seed = (ncptl_int) rndseed_int;
-}
-(void) ncptl_seed_random_task(random_seed, (ncptl_int)physrank);
-
- /* Establish a mapping from (virtual) task IDs to (physical) ranks. */
-procmap = ncptl_allocate_task_map (var_num_tasks);
-virtrank = ncptl_physical_to_virtual (procmap, physrank);
-
- /* Perform initializations specific to the c_mpi backend. */
-ncptl_log_add_comment ("MPI send routines", "MPI_Send() and MPI_Isend()");
-ncptl_log_add_comment ("MPI reduction operation", REDUCE_OPERATION_NAME);
-sprintf (log_key_str, "[0, %" NICS "]", mpi_tag_ub);
-ncptl_log_add_comment ("MPI tag range", log_key_str);
-
- /* Open the log file and write some standard prologue information to it. */
-logstate = ncptl_log_open (logfiletmpl, physrank);
-ncptl_log_write_prologue (logstate, argv[0], logfile_uuid, "c_mpi", "C + MPI",
-var_num_tasks,
-arguments, sizeof(arguments)/sizeof(NCPTL_CMDLINE),
-sourcecode);
-ncptl_free (logfile_uuid);
-
- /* Allocate a variety of dynamically growing queues. */
-eventqueue = ncptl_queue_init (sizeof (CONC_EVENT));
-touchedqueue = ncptl_queue_init (sizeof (ncptl_int));
-
- /* Perform initializations specific to the c_mpi backend. */
-sendreqQ = ncptl_queue_init (sizeof (MPI_Request));
-sendstatQ = ncptl_queue_init (sizeof (MPI_Status));
-recvreqQ = ncptl_queue_init (sizeof (MPI_Request));
-recvstatQ = ncptl_queue_init (sizeof (MPI_Status));
-communicators = ncptl_set_init (ESTIMATED_COMMUNICATORS, var_num_tasks*sizeof(char), sizeof(MPI_Comm));procflags = (char *) ncptl_malloc (var_num_tasks*sizeof(char), 0);
-for (i=0; i<var_num_tasks; i++)
-procflags[i] = 1;
-ncptl_set_insert (communicators, (void *)procflags, (void *)&comm_world);
-ncptl_free (procflags);
-
- /****************************************************
-  * Generated, program-specific initialization code. *
-  ****************************************************/
-
- /* FOR EACH var_trial IN [(['1LL'], 'var_numtrials')]... */
-{
-LOOPBOUNDS loopbounds[1];   /* List of range descriptions */
-ncptl_int rangenum;   /* Current offset into loopbounds[] */
-ncptl_int initial_vals[1];   /* Cache of the initial, enumerated values */
-ncptl_int final_val;   /* Cache of the final value */
-
- /* Write range 0's loop bounds, "next" function, and termination function to loopbounds[0]. */
-loopbounds[0].list_comp = NULL;
-initial_vals[0] = 1LL;
-final_val = var_numtrials;
-loopbounds[0].integral = 1;
-loopbounds[0].u.i.startval = initial_vals[0];
-loopbounds[0].u.i.endval = final_val;
-loopbounds[0].comparator = initial_vals[0]<=final_val ? CONC_LEQ : CONC_GEQ;
-loopbounds[0].increment = CONC_ADD;
-loopbounds[0].u.i.incval = initial_vals[0]<=final_val ? 1LL : -1LL;
-
- /* Now that we've defined all of our ranges we iterate over each range
-  * and each element within each range. */
-for (rangenum=0LL; rangenum<1; rangenum++) {
-LOOPBOUNDS * thisrange = &loopbounds[rangenum];   /* Current range */
-if (conc_seq_nonempty (thisrange))
-for (conc_seq_init (thisrange);
-conc_seq_continue (thisrange);
-conc_seq_next (thisrange)) {
-ncptl_int var_trial;
-if (thisrange->list_comp == NULL)
- /* The loopvar field contains the value for var_trial. */
-var_trial = thisrange->integral ? thisrange->u.i.loopvar : CONC_DBL2INT(thisrange->u.d.loopvar);
-else
- /* The queue of list-comprehension values provides the value for var_trial. */
-var_trial = *(ncptl_int *)ncptl_queue_pop(thisrange->list_comp);
-{
- /* TASK 0LL ARE ASSIGNED PROCESSOR 0LL */
-{
- /* In the following, "task ID" implies a virtual rank in the
-  * computation while "processor" implies a physical rank. */
-ncptl_int virtID = 0LL;   /* Task ID */
-procmap = ncptl_conditionally_copy_task_map (procmap);
-virtrank = ncptl_assign_processor (virtID, 0LL, procmap, physrank);
-}
- /* THEN... */
- /* TASK 0LL OUTPUTS ['"Testing random bisection pattern "', 'var_trial', '"/"', 'var_numtrials'] */
-if ((0LL) == virtrank) {   /* TASK 0LL */
- /* The current coNCePTuaL statement applies to our task. */
-CONC_EVENT *thisev = conc_allocate_event (EV_CODE);
-thisev->s.code.number = 0;
-thisev->s.code.procmap = NULL;
-thisev->s.code.var_numtrials = var_numtrials;
-thisev->s.code.var_trial = var_trial;
-}
- /* THEN... */
- /* ALL TASKS ARE ASSIGNED PROCESSOR [random] */
-{
- /* In the following, "task ID" implies a virtual rank in the
-  * computation while "processor" implies a physical rank. */
-ncptl_int virtID;   /* Task ID */
-procmap = ncptl_conditionally_copy_task_map (procmap);
-for (virtID=0; virtID<var_num_tasks; virtID++) {
-virtrank = ncptl_assign_processor (virtID, ncptl_random_task(0, var_num_tasks-1, -1), procmap, physrank);
-}
-}
- /* THEN... */
- /* TASKS var_src SUCH THAT ((var_src)&1)==0 RECEIVES FROM TASK (var_src)+(1LL) */
-{
-ncptl_int var_src;
- /* Loop over all tasks to see which will send to us. */
-for (var_src=0; var_src<var_num_tasks; var_src++)
-if (((var_src)&1)==0) {
- /* var_src now represents one of the tasks that will send to us. */
-ncptl_int ivar_a_task = var_src;
-if (virtrank == ((var_src)+(1LL))) {
- /* In this scope, we must be a message recipient. */
- /* Prepare to receive var_wups messages. */
-ncptl_int numreps = var_wups;   /* Number of messages */
-if (numreps > 1LL) {
-CONC_EVENT * repeatev;   /* Event specifying the number of repetitions to perform */
-repeatev = conc_allocate_event (EV_REPEAT);
-repeatev->s.rep.end_event = ncptl_queue_length(eventqueue);
-repeatev->s.rep.numreps = numreps;
-}
-
- /* Ensure we have at least one message to receive. */
-if (numreps > 0LL) {
-CONC_EVENT *thisev = conc_allocate_event (EV_RECV);
-
- /* Fill in all of the fields of a receive-event structure. */
-thisev->s.recv.source = ncptl_virtual_to_physical (procmap, ivar_a_task);
-thisev->s.recv.size = var_msgsize;
-thisev->s.recv.alignment = 0LL;
-thisev->s.recv.misaligned = 0;
-thisev->s.recv.touching = 0;
-thisev->s.recv.verification = 0;
-thisev->s.recv.tag = 0LL;
-thisev->s.recv.pendingsends = pendingsends;
-thisev->s.recv.pendingrecvs = pendingrecvs;
-thisev->s.recv.bufferofs = 0LL;
-thisev->s.recv.buffernum = thisev->s.recv.verification ? pendingsends+pendingrecvs : 0;
-(void) ncptl_malloc_message (thisev->s.recv.size+thisev->s.recv.bufferofs, thisev->s.recv.alignment, thisev->s.recv.buffernum, thisev->s.recv.misaligned);
-thisev->s.recv.buffer = NULL;
-thisev->s.recv.tag = map_tag_into_MPI_range (thisev->s.recv.tag);
-}
-}
-}
-}
- /* TASKS var_src SUCH THAT ((var_src)&1)==0 SENDS TO TASK (var_src)+(1LL) */
-{
-ncptl_int var_src = virtrank;
-if (((var_src)&1)==0) {   /* TASKS var_src SUCH THAT ((var_src)&1)==0 */
- /* The current coNCePTuaL statement applies to our task. */
-ncptl_int virtdest = (var_src)+(1LL);
-if (virtdest>=0 && virtdest<var_num_tasks) {
- /* In this scope, (var_src)+(1LL) represents a single receiver. */
- /* Prepare to send var_wups messages. */
-ncptl_int numreps = var_wups;   /* Number of messages */
-if (numreps > 1LL) {
-CONC_EVENT * repeatev;   /* Event specifying the number of repetitions to perform */
-repeatev = conc_allocate_event (EV_REPEAT);
-repeatev->s.rep.end_event = ncptl_queue_length(eventqueue);
-repeatev->s.rep.numreps = numreps;
-}
-
- /* Ensure we have at least one message to send. */
-if (numreps > 0LL) {
-CONC_EVENT *thisev = conc_allocate_event (EV_SEND);
-if (virtrank == ((var_src)+(1LL)))
-ncptl_fatal ("Send-to-self deadlock encountered on task %d in line 21 of the source code", virtrank);
-
- /* Fill in all of the fields of a send-event structure. */
-thisev->s.send.dest = ncptl_virtual_to_physical (procmap, (var_src)+(1LL));
-thisev->s.send.size = var_msgsize;
-thisev->s.send.alignment = 0LL;
-thisev->s.send.misaligned = 0;
-thisev->s.send.touching = 0;
-thisev->s.send.verification = 0;
-thisev->s.send.tag = 0LL;
-thisev->s.send.pendingsends = pendingsends;
-thisev->s.send.pendingrecvs = pendingrecvs;
-thisev->s.send.bufferofs = 0LL;
-thisev->s.send.buffernum = thisev->s.send.verification ? pendingsends+pendingrecvs : 0;
-(void) ncptl_malloc_message (thisev->s.send.size+thisev->s.send.bufferofs, thisev->s.send.alignment, thisev->s.send.buffernum, thisev->s.send.misaligned);
-thisev->s.send.buffer = NULL;
-thisev->s.send.tag = map_tag_into_MPI_range (thisev->s.send.tag);
-}
-}
-}
-}
- /* THEN... */
- /* ALL TASKS SYNCHRONIZE */
-{   /* ALL TASKS */
- /* The current coNCePTuaL statement applies to our task. */
-CONC_EVENT *thisev = conc_allocate_event (EV_SYNC);
-thisev->s.sync.communicator = MPI_COMM_WORLD;
-}
- /* THEN... */
- /* ALL TASKS RESET THEIR COUNTERS */
-{   /* ALL TASKS */
- /* The current coNCePTuaL statement applies to our task. */
-(void) conc_allocate_event (EV_RESET);
-}
- /* THEN... */
- /* TASKS var_src SUCH THAT ((var_src)&1)==0 RECEIVES FROM TASK (var_src)+(1LL) */
-{
-ncptl_int var_src;
- /* Loop over all tasks to see which will send to us. */
-for (var_src=0; var_src<var_num_tasks; var_src++)
-if (((var_src)&1)==0) {
- /* var_src now represents one of the tasks that will send to us. */
-ncptl_int ivar_b_task = var_src;
-if (virtrank == ((var_src)+(1LL))) {
- /* In this scope, we must be a message recipient. */
- /* Prepare to receive var_nummsgs messages. */
-ncptl_int numreps = var_nummsgs;   /* Number of messages */
-if (numreps > 1LL) {
-CONC_EVENT * repeatev;   /* Event specifying the number of repetitions to perform */
-repeatev = conc_allocate_event (EV_REPEAT);
-repeatev->s.rep.end_event = ncptl_queue_length(eventqueue);
-repeatev->s.rep.numreps = numreps;
-}
-
- /* Ensure we have at least one message to receive. */
-if (numreps > 0LL) {
-CONC_EVENT *thisev = conc_allocate_event (EV_RECV);
-
- /* Fill in all of the fields of a receive-event structure. */
-thisev->s.recv.source = ncptl_virtual_to_physical (procmap, ivar_b_task);
-thisev->s.recv.size = var_msgsize;
-thisev->s.recv.alignment = 0LL;
-thisev->s.recv.misaligned = 0;
-thisev->s.recv.touching = 0;
-thisev->s.recv.verification = 0;
-thisev->s.recv.tag = 0LL;
-thisev->s.recv.pendingsends = pendingsends;
-thisev->s.recv.pendingrecvs = pendingrecvs;
-thisev->s.recv.bufferofs = 0LL;
-thisev->s.recv.buffernum = thisev->s.recv.verification ? pendingsends+pendingrecvs : 0;
-(void) ncptl_malloc_message (thisev->s.recv.size+thisev->s.recv.bufferofs, thisev->s.recv.alignment, thisev->s.recv.buffernum, thisev->s.recv.misaligned);
-thisev->s.recv.buffer = NULL;
-thisev->s.recv.tag = map_tag_into_MPI_range (thisev->s.recv.tag);
-}
-}
-}
-}
- /* TASKS var_src SUCH THAT ((var_src)&1)==0 SENDS TO TASK (var_src)+(1LL) */
-{
-ncptl_int var_src = virtrank;
-if (((var_src)&1)==0) {   /* TASKS var_src SUCH THAT ((var_src)&1)==0 */
- /* The current coNCePTuaL statement applies to our task. */
-ncptl_int virtdest = (var_src)+(1LL);
-if (virtdest>=0 && virtdest<var_num_tasks) {
- /* In this scope, (var_src)+(1LL) represents a single receiver. */
- /* Prepare to send var_nummsgs messages. */
-ncptl_int numreps = var_nummsgs;   /* Number of messages */
-if (numreps > 1LL) {
-CONC_EVENT * repeatev;   /* Event specifying the number of repetitions to perform */
-repeatev = conc_allocate_event (EV_REPEAT);
-repeatev->s.rep.end_event = ncptl_queue_length(eventqueue);
-repeatev->s.rep.numreps = numreps;
-}
-
- /* Ensure we have at least one message to send. */
-if (numreps > 0LL) {
-CONC_EVENT *thisev = conc_allocate_event (EV_SEND);
-if (virtrank == ((var_src)+(1LL)))
-ncptl_fatal ("Send-to-self deadlock encountered on task %d in line 24 of the source code", virtrank);
-
- /* Fill in all of the fields of a send-event structure. */
-thisev->s.send.dest = ncptl_virtual_to_physical (procmap, (var_src)+(1LL));
-thisev->s.send.size = var_msgsize;
-thisev->s.send.alignment = 0LL;
-thisev->s.send.misaligned = 0;
-thisev->s.send.touching = 0;
-thisev->s.send.verification = 0;
-thisev->s.send.tag = 0LL;
-thisev->s.send.pendingsends = pendingsends;
-thisev->s.send.pendingrecvs = pendingrecvs;
-thisev->s.send.bufferofs = 0LL;
-thisev->s.send.buffernum = thisev->s.send.verification ? pendingsends+pendingrecvs : 0;
-(void) ncptl_malloc_message (thisev->s.send.size+thisev->s.send.bufferofs, thisev->s.send.alignment, thisev->s.send.buffernum, thisev->s.send.misaligned);
-thisev->s.send.buffer = NULL;
-thisev->s.send.tag = map_tag_into_MPI_range (thisev->s.send.tag);
-}
-}
-}
-}
- /* THEN... */
- /* ALL TASKS SYNCHRONIZE */
-{   /* ALL TASKS */
- /* The current coNCePTuaL statement applies to our task. */
-CONC_EVENT *thisev = conc_allocate_event (EV_SYNC);
-thisev->s.sync.communicator = MPI_COMM_WORLD;
-}
- /* THEN... */
- /* ALL TASKS LOGS "Bandwidth (MiB/s)" */
-{   /* ALL TASKS */
- /* The current coNCePTuaL statement applies to our task. */
-CONC_EVENT *thisev = conc_allocate_event (EV_CODE);
-thisev->s.code.number = 1;
-thisev->s.code.procmap = NULL;
-thisev->s.code.var_rounding = var_rounding;
-}
-}
-}
-}
-}
- /* Begin a new top-level statement. */
-(void) conc_allocate_event (EV_NEWSTMT);
-
- /*************************
-  * More boilerplate code *
-  *************************/
-
- /* Abort if the program will terminate with pending messages. */
-if (pendingsends && pendingrecvs)
-ncptl_fatal("Neglected to await the completion of %" NICS " asynchronous %s and %" NICS " asynchronous %s",
-pendingsends, pendingsends==1LL ? "send" : "sends",
-pendingrecvs, pendingrecvs==1LL ? "receive" : "receives");
-else
-if (pendingsends)
-ncptl_fatal("Neglected to await the completion of %" NICS " asynchronous %s",
-pendingsends, pendingsends==1LL ? "send" : "sends");
-else
-if (pendingrecvs)
-ncptl_fatal("Neglected to await the completion of %" NICS " asynchronous %s",
-pendingrecvs, pendingrecvs==1LL ? "receive" : "receives");
-
- /* Allocate memory for non-unique messages and asynchronous
-  * message handles now that we know how much memory we need
-  * to allocate. */
-eventlist = (CONC_EVENT *) ncptl_queue_contents (eventqueue, 0);
-numevents = ncptl_queue_length (eventqueue);
-sendrequests = (MPI_Request *) ncptl_queue_contents (sendreqQ, 0);
-recvrequests = (MPI_Request *) ncptl_queue_contents (recvreqQ, 0);
-for (i=0; i<numevents; i++) {
-CONC_EVENT *thisev = &eventlist[i];   /* Cache of the current event */
-switch (thisev->type) {
-case EV_SEND:
-if (!thisev->s.send.buffer)
-thisev->s.send.buffer = ncptl_malloc_message (thisev->s.send.bufferofs + thisev->s.send.size,
-thisev->s.send.alignment,
-thisev->s.send.buffernum,
-thisev->s.send.misaligned);
-if (thisev->s.send.verification)
-ncptl_fill_buffer (CONC_GETBUFPTR(send), thisev->s.send.bufferofs + thisev->s.send.size, -1);
-break;
-
-case EV_RECV:
-if (!thisev->s.recv.buffer)
-thisev->s.recv.buffer = ncptl_malloc_message (thisev->s.recv.bufferofs + thisev->s.recv.size,
-thisev->s.recv.alignment,
-thisev->s.recv.buffernum,
-thisev->s.recv.misaligned);
-if (thisev->s.recv.verification)
-ncptl_fill_buffer (CONC_GETBUFPTR(recv), thisev->s.recv.bufferofs + thisev->s.recv.size, -1);
-break;
-
-default:
-break;
-}
-}
-}
-
-/* Process a subset of the events in a given event list. */
-static void conc_process_events (CONC_EVENT *eventlist,
-ncptl_int firstev, ncptl_int lastev, ncptl_int numreps)
-{
-CONC_EVENT * thisev;   /* Cache of the current event */
-CONC_EVENT * thisev_first = &eventlist[firstev];   /* Cache of the first event */
-ncptl_int i;   /* Iterate over events. */
-ncptl_int j;   /* Iterate over repetitions. */
-
- /* Declarations specific to the c_mpi backend */
-MPI_Status status;   /* Not needed but required by MPI_Recv() */
-
- /* Process from event firstev to event lastev (both inclusive). */
-for (j=numreps; j>0; j--)
-for (i=firstev, thisev=thisev_first; i<=lastev; i++, thisev++) {
- /* Declare variables needed by all C-based backends. */
-
- /* Process a single event. */
-switch (thisev->type) {
-case EV_SEND:
- /* Synchronous send */
-(void) MPI_Send (CONC_GETBUFPTR(send),
-(int)thisev->s.send.size, MPI_BYTE,
-(int)thisev->s.send.dest, (int)thisev->s.send.tag, MPI_COMM_WORLD);
-var_total_bytes += thisev->s.send.size;
-break;
-
-case EV_RECV:
- /* Synchronous receive */
-(void) MPI_Recv (CONC_GETBUFPTR(recv),
-(int)thisev->s.recv.size, MPI_BYTE,
-(int)thisev->s.recv.source, (int)thisev->s.recv.tag,
-MPI_COMM_WORLD, &status);
-var_total_bytes += thisev->s.recv.size;
-break;
-
-case EV_SYNC:
- /* Synchronize a subset of the tasks. */
-(void) MPI_Barrier (thisev->s.sync.communicator);
-break;
-
-case EV_RESET:
- /* Reset all of the counters exported to coNCePTuaL programs. */
-var_total_bytes = 0;
-starttime = ncptl_time();
-break;
-
-case EV_REPEAT:
- /* Repeatedly perform the next batch of events. */
-conc_process_events (eventlist, i+1, thisev->s.rep.end_event, thisev->s.rep.numreps);
-i = thisev->s.rep.end_event;
-thisev = &eventlist[i];
-break;
-
-case EV_NEWSTMT:
- /* Begin a new table in the log file. */
-if (!suppress_output) {
-uint64_t stop_elapsed_usecs = ncptl_time();
-var_elapsed_usecs = stop_elapsed_usecs - starttime;
-ncptl_log_commit_data (logstate);
-starttime += ncptl_time() - stop_elapsed_usecs;
-}
-break;
-
-case EV_CODE:
- /* Execute an arbitrary piece of code. */
-switch (thisev->s.code.number) {
-case 0:
- /* TASK 0LL OUTPUTS ['"Testing random bisection pattern "', 'var_trial', '"/"', 'var_numtrials'] */
-if (!suppress_output) {
-uint64_t stop_elapsed_usecs = ncptl_time();
-var_elapsed_usecs = stop_elapsed_usecs - starttime;
-printf ("%s%.10lg%s%.10lg\n", "Testing random bisection pattern ", (double)thisev->s.code.var_trial, "/", (double)thisev->s.code.var_numtrials);
-fflush (stdout);
-starttime += ncptl_time() - stop_elapsed_usecs;
-}
-break;
-
-case 1:
- /* ALL TASKS LOGS "Bandwidth (MiB/s)" */
-if (!suppress_output) {
-uint64_t stop_elapsed_usecs = ncptl_time();
-var_elapsed_usecs = stop_elapsed_usecs - starttime;
-ncptl_log_write (logstate, 0, "Bandwidth (MiB/s)", NCPTL_FUNC_HISTOGRAM, 0.0, ((double)thisev->s.code.var_rounding)*(ncptl_dfunc_round(((((double)var_total_bytes)*(1000000.0))/(((double)var_elapsed_usecs)*(1048576.0)))/((double)thisev->s.code.var_rounding))));
-starttime += ncptl_time() - stop_elapsed_usecs;
-}
-break;
-
-default:
- /* The C code generation module must be broken. */
-ncptl_fatal ("Internal error: unknown EV_CODE block %" NICS, thisev->s.code.number);
-break;
-}
-break;
-
-default:
- /* The c_generic backend or the c_mpi backend must be broken. */
-ncptl_fatal ("Internal error: unknown event type %d", thisev->type);
-break;
-}
-}
-}
-
-/* Finish up cleanly and return a status code. */
-static int conc_finalize (void)
-{
-int exitcode = 0;   /* Program exit code (to pass to exit()) */
-
- /* Declarations specific to the c_mpi backend */
-int mpiresult;   /* Return code from MPI_Finalize() */
-char log_key_str[128];   /* String representing the range of valid MPI tags */
-
- /* Finalization code specific to the c_mpi backend */
-log_key_str[0] = '\0';
-if (conc_mcast_tallies[CONC_MCAST_MPI_BCAST] > 0) {
-char onefuncstr[50];
-sprintf (onefuncstr, "%sMPI_Bcast()*%" NICS,
-log_key_str[0] == '\0' ? "" : " ", conc_mcast_tallies[CONC_MCAST_MPI_BCAST]);
-strcat (log_key_str, onefuncstr);
-}
-if (conc_mcast_tallies[CONC_MCAST_MPI_ALLTOALL] > 0) {
-char onefuncstr[50];
-sprintf (onefuncstr, "%sMPI_Alltoall()*%" NICS,
-log_key_str[0] == '\0' ? "" : " ", conc_mcast_tallies[CONC_MCAST_MPI_ALLTOALL]);
-strcat (log_key_str, onefuncstr);
-}
-if (conc_mcast_tallies[CONC_MCAST_MPI_ALLTOALLV] > 0) {
-char onefuncstr[50];
-sprintf (onefuncstr, "%sMPI_Alltoallv()*%" NICS,
-log_key_str[0] == '\0' ? "" : " ", conc_mcast_tallies[CONC_MCAST_MPI_ALLTOALLV]);
-strcat (log_key_str, onefuncstr);
-}
-if (log_key_str[0] != '\0')
-ncptl_log_add_comment ("Multicast functions used (statically)", log_key_str);
-
- /* Write a standard epilogue to the log file. */
-ncptl_log_commit_data (logstate);
-ncptl_log_write_epilogue (logstate);
-ncptl_log_close (logstate);
-
- /* Inform the run-time library that it's no longer needed. */
-ncptl_queue_empty (eventqueue);
-ncptl_free (eventqueue);
-ncptl_finalize();
-
- /* Finalization code specific to the c_mpi backend */
-mpiresult = MPI_Finalize();
-mpi_is_running = 0;
-exitcode = mpiresult!=MPI_SUCCESS;
-
- /* Return an exit status code. */
-return exitcode;
-}
-
-/*************************************************************************/
-/***************************** MAIN ROUTINE ******************************/
-/*************************************************************************/
-
-/* Program execution starts here. */
-int main (int argc, char *argv[])
-{
- /* Declare variables needed by all C-based backends. */
-CONC_EVENT * eventlist;   /* List of events to execute */
-ncptl_int numevents;   /* Number of entries in eventlist[] */
-
- /* ----- Initialization ----- */
-conc_initialize (argc, argv);
-eventlist = (CONC_EVENT *) ncptl_queue_contents (eventqueue, 0);
-numevents = ncptl_queue_length (eventqueue);
-sendrequests = (MPI_Request *) ncptl_queue_contents (sendreqQ, 0);
-sendstatuses = (MPI_Status *) ncptl_queue_contents (sendstatQ, 0);
-recvrequests = (MPI_Request *) ncptl_queue_contents (recvreqQ, 0);
-recvstatuses = (MPI_Status *) ncptl_queue_contents (recvstatQ, 0);
-starttime = ncptl_time();
-
- /* ----- Event-list processing ----- */
-conc_process_events (eventlist, 0, numevents-1, 1);
-
- /* ----- Finalization ----- */
-return conc_finalize();
-}

From 7c4cae0a9ce406b397ca0da8d3814cc4335b75fe Mon Sep 17 00:00:00 2001
From: Xin Wang <xwang149@hawk.iit.edu>
Date: Tue, 22 Jan 2019 15:09:46 -0600
Subject: [PATCH 03/15] Delete bisect.ncptl

---
 scripts/conceptual_benchmarks/bisect.ncptl | 27 ----------------------
 1 file changed, 27 deletions(-)
 delete mode 100644 scripts/conceptual_benchmarks/bisect.ncptl

diff --git a/scripts/conceptual_benchmarks/bisect.ncptl b/scripts/conceptual_benchmarks/bisect.ncptl
deleted file mode 100644
index 950fd26f..00000000
--- a/scripts/conceptual_benchmarks/bisect.ncptl
+++ /dev/null
@@ -1,27 +0,0 @@
-###################################################
-# Measure random bisection-bandwidth patterns     #
-# By Scott Pakin <pakin@lanl.gov>                 #
-#                                                 #
-# Inspired by Hoefler, Schneider, and Lumsdaine's #
-# "Multistage Switches are not Crossbars" paper   #
-###################################################
-
-Require language version "1.5".
-
-nummsgs is "Number of messages per trial" and comes from "--nummsgs" or "-n" with default 100.
-wups is "Number of warmup messages" and comes from "--wups" or "-w" with default 3.
-msgsize is "Message size in bytes" and comes from "--bytes" or "-b" with default 1M.
-numtrials is "Number of bisection patterns" and comes from "--trials" or "-t" with default 5000.
-rounding is "Round measurements to the nearest N" and comes from "--round" or "-r" with default 5.
-
-For each trial in {1, ..., numtrials} {
-  task 0 is assigned to processor 0 then
-  task 0 outputs "Testing random bisection pattern " and trial and "/" and numtrials then
-  all tasks are assigned to a random processor then
-  tasks src such that src is even send wups msgsize-byte messages to task src+1 then
-  all tasks synchronize then
-  all tasks reset their counters then
-  tasks src such that src is even send nummsgs msgsize-byte messages to task src+1 then
-  all tasks synchronize then
-  all tasks log a histogram of rounding*round(total_bytes*1E6/(elapsed_usecs*1M)/rounding) as "Bandwidth (MiB/s)"
-}

From 15f3c1b66d5bce319eea463c4c291a824cde848b Mon Sep 17 00:00:00 2001
From: Xin Wang <xwang149@hawk.iit.edu>
Date: Tue, 22 Jan 2019 15:10:05 -0600
Subject: [PATCH 04/15] Delete latency.ncptl

---
 scripts/conceptual_benchmarks/latency.ncptl | 24 ---------------------
 1 file changed, 24 deletions(-)
 delete mode 100644 scripts/conceptual_benchmarks/latency.ncptl

diff --git a/scripts/conceptual_benchmarks/latency.ncptl b/scripts/conceptual_benchmarks/latency.ncptl
deleted file mode 100644
index 4c4b14d9..00000000
--- a/scripts/conceptual_benchmarks/latency.ncptl
+++ /dev/null
@@ -1,24 +0,0 @@
-# A ping-pong latency test written in coNCePTuaL
-
-Require language version "1.5".
-
-# Parse the command line.
-reps is "Number of repetitions of each message size" and comes from
- "--reps" or "-r" with default 1000.
-maxbytes is "Maximum number of bytes to transmit" and comes from
- "--maxbytes" or "-m" with default 1M.
-
-# Ensure that we have a peer with whom to communicate.
-Assert that "the latency test requires at least two tasks" with num_tasks>=2.
-
-# Perform the benchmark.
-For each msgsize in {0}, {1, 2, 4, ..., maxbytes} {
-  for reps repetitions {
-    task 0 resets its counters then
-    task 0 sends a msgsize byte message to task 1 then
-    task 1 sends a msgsize byte message to task 0 then
-    task 0 logs the msgsize as "Bytes" and
-                the median of elapsed_usecs/2 as "1/2 RTT (usecs)"
-  } then
-  task 0 computes aggregates
-}

From 64b3e69ca6f95c7bcaaf240f3fc449ce2e07d1bb Mon Sep 17 00:00:00 2001
From: Xin Wang <xwang149@hawk.iit.edu>
Date: Tue, 22 Jan 2019 15:10:22 -0600
Subject: [PATCH 05/15] Delete cosmoflow.c

---
 scripts/conceptual_benchmarks/cosmoflow.c | 1317 ---------------------
 1 file changed, 1317 deletions(-)
 delete mode 100644 scripts/conceptual_benchmarks/cosmoflow.c

diff --git a/scripts/conceptual_benchmarks/cosmoflow.c b/scripts/conceptual_benchmarks/cosmoflow.c
deleted file mode 100644
index dcca59ae..00000000
--- a/scripts/conceptual_benchmarks/cosmoflow.c
+++ /dev/null
@@ -1,1317 +0,0 @@
-/**********************************************************************
- * This file was generated by coNCePTuaL on Mon Jan  7 23:10:01 2019
- * using the c_mpi backend (C + MPI).
- * Do not modify this file; modify /Users/xin/macworkspace/codes-dev/codes/scripts/conceptual_benchmarks/cosmoflow.ncptl instead.
- *
- * Entire source program
- * ---------------------
- *   # Measure the performance of MPI_Allgather()
- *   # By Scott Pakin <pakin@lanl.gov>
- *   #
- *   # N.B. Requires the c_mpi backend.
- *   Require language version "1.5".
- *   # Parse the command line.
- *   numwords is "Message size (words)" and comes from "--msgsize" or "-s" with default 28825K.
- *   reps is "Number of repetitions" and comes from "--reps" or "-r" with default 100.
- *   computetime is "Computation time (ms)" and comes from "--compute" or "-c" with default 129.
- *   
- *   # Allocate a send buffer and a receive buffer.
- *   Task 0 multicasts a numwords*num_tasks word message from buffer 0 to all other tasks.
- *   Task 0 multicasts a numwords*num_tasks word message from buffer 1 to all other tasks.
- *   
- *   # Measure the performance of MPI_Allreduce().
- *   Task 0 resets its counters then
- *   for reps repetitions {
- *       all tasks COMPUTES FOR computetime MILLISECONDS then
- *       all tasks backend execute "
- *           MPI_Allreduce([MESSAGE BUFFER 0], [MESSAGE BUFFER 1], (int)" and numwords and ",
- *                   MPI_INT, MPI_SUM, MPI_COMM_WORLD);
- *           " then
- *       all tasks backend execute "
- *           MPI_Allreduce([MESSAGE BUFFER 0], [MESSAGE BUFFER 1], (int)" and numwords and ",
- *                   MPI_INT, MPI_SUM, MPI_COMM_WORLD);
- *           "
- *   } then
- *   task 0 logs elapsed_usecs/1000 as "Elapse time (ms)".
- **********************************************************************/
-
-/*****************
- * Include files *
- *****************/
-
-/* Header files needed by all C-based backends */
-#include <stdio.h>
-#include <string.h>
-#include <ncptl/ncptl.h>
-
-/* Header files specific to the c_mpi backend */
-#include <mpi.h>
-#include <stdarg.h>
-
-/**********
- * Macros *
- **********/
-
-/* Define the maximum loop trip count that we're willing to unroll fully. */
-#define CONC_MAX_UNROLL 5
-
-/* Specify the minimum number of trial iterations in each FOR <time> loop. */
-#define CONC_FOR_TIME_TRIALS 1
-
-/* Define a macro that rounds a double to a ncptl_int. */
-#define CONC_DBL2INT(D) ((ncptl_int)((D)+0.5))
-
-/* Define a macro that increments a buffer pointer by a byte offset. */
-#define CONC_GETBUFPTR(S) ((void *)((char *)thisev->s.S.buffer + thisev->s.S.bufferofs))
-
-/* Implement ncptl_func_task_of in terms of ncptl_physical_to_virtual. */
-#define ncptl_func_task_of(P) ((ncptl_int)(P) < 0LL || (ncptl_int)(P) >= var_num_tasks ? -1LL : ncptl_physical_to_virtual (procmap, (ncptl_int)(P)))
-#define ncptl_dfunc_task_of(P) ((double) ncptl_func_task_of(P))
-
-/* Implement ncptl_func_processor_of in terms of ncptl_virtual_to_physical. */
-#define ncptl_func_processor_of(V) ((ncptl_int)(V) < 0LL || (ncptl_int)(V) >= var_num_tasks ? -1LL : ncptl_virtual_to_physical (procmap, (ncptl_int)(V)))
-#define ncptl_dfunc_processor_of(V) ((double) ncptl_func_processor_of(V))
-
-/* Define a macro that increments REDUCE's alternate buffer pointer by a byte offset. */
-#define CONC_GETALTBUFPTR(S) ((void *)((char *)thisev->s.S.altbuffer + thisev->s.S.bufferofs))
-
-/* Estimate the number of unique communicators that this program will need.
- * (The tradeoff is one of initialization time versus memory consumption.) */
-#define ESTIMATED_COMMUNICATORS 128
-
-/* Specify an operation to use for all reduction operations. */
-#define REDUCE_OPERATION MPI_SUM
-#define REDUCE_OPERATION_NAME "MPI_SUM"
-
-
-/*********************
- * Type declarations *
- *********************/
-
-/* Enumerate the various mechanisms used to implement MULTICAST statements. */
-typedef enum {
-CONC_MCAST_MPI_BCAST,       /* One to many */
-CONC_MCAST_MPI_ALLTOALL,    /* Many to many, same data to all */ 
-CONC_MCAST_MPI_ALLTOALLV,   /* General many to many */
-CONC_MCAST_MPI_NUM_FUNCS    /* Number of the above */
-} CONC_MCAST_MPI_FUNC;
-
-/* Define the type of event to perform. */
-typedef enum {
-EV_SEND,     /* Synchronous send */
-EV_ASEND,    /* Asynchronous send */
-EV_RECV,     /* Synchronous receive */
-EV_ARECV,    /* Asynchronous receive */
-EV_WAIT,     /* Wait for all asynchronous sends/receives to complete */
-EV_DELAY,    /* Spin or sleep */
-EV_TOUCH,    /* Touch a region of memory */
-EV_SYNC,     /* Barrier synchronization */
-EV_RESET,    /* Reset counters */
-EV_STORE,    /* Store all counters' current values */
-EV_RESTORE,  /* Restore the previously pushed counter values */
-EV_FLUSH,    /* Compute aggregate functions for log-file columns */
-EV_MCAST,    /* Synchronous multicast */
-EV_REDUCE,   /* Reduction with or without a subsequent multicast */
-EV_BTIME,    /* Beginning of a timed loop */
-EV_ETIME,    /* Ending of a timed loop */
-EV_REPEAT,   /* Repeatedly process the next N events */
-EV_SUPPRESS, /* Suppress writing to the log and standard output */
-EV_NEWSTMT,  /* Beginning of a new top-level statement */
-EV_CODE,     /* None of the above */
-NUM_EVS      /* Number of event types in CONC_EVENT_TYPE */
-} CONC_EVENT_TYPE;
-
-/* Describe a synchronous or asynchronous send event. */
-typedef struct {
-ncptl_int dest;         /* Destination task */
-ncptl_int size;         /* Number of bytes to send */
-ncptl_int alignment;    /* Message alignment (in bytes) */
-ncptl_int pendingsends; /* # of outstanding sends */
-ncptl_int pendingrecvs; /* # of outstanding receives */
-ncptl_int buffernum;    /* Buffer # to send from */
-ncptl_int bufferofs;    /* Byte offset into the message buffer */
-ncptl_int tag;          /* Tag to use for selective receives */
-int misaligned;         /* 1=misaligned from a page; 0=align as specified */
-int touching;           /* 1=touch every word before sending */
-int verification;       /* 1=fill message buffer with known contents */
-void *buffer;           /* Pointer to message memory */
-MPI_Request * handle;   /* MPI handle representing an asynchronous send */
-} CONC_SEND_EVENT;
-
-/* Describe a synchronous or asynchronous receive event. */
-typedef struct {
-ncptl_int source;       /* Source task */
-ncptl_int size;         /* Number of bytes to receive */
-ncptl_int alignment;    /* Message alignment (in bytes) */
-ncptl_int pendingsends; /* # of outstanding sends */
-ncptl_int pendingrecvs; /* # of outstanding receives */
-ncptl_int buffernum;    /* Buffer # to receive into */
-ncptl_int bufferofs;    /* Byte offset into the message buffer */
-ncptl_int tag;          /* Tag to use for selective receives */
-int misaligned;         /* 1=misaligned from a page; 0=align as specified */
-int touching;           /* 1=touch every word after reception */
-int verification;       /* 1=verify that all bits are correct */
-void *buffer;           /* Pointer to message memory */
-MPI_Request * handle;   /* MPI handle representing an asynchronous receive */
-} CONC_RECV_EVENT;
-
-/* Describe a wait-for-asynchronous-completions event. */
-typedef struct {
-ncptl_int numsends;     /* # of sends we expect to complete. */
-ncptl_int numrecvs;     /* # of receives we expect to complete. */
-ncptl_int numrecvbytes; /* # of bytes we expect to receive-complete */
-ncptl_int *touchedlist;    /* List of receives that need to be touched */
-ncptl_int numtouches;         /* # of elements in the above */
-} CONC_WAIT_EVENT;
-
-/* Describe a spin or sleep delay. */
-typedef struct {
-uint64_t microseconds;  /* Length of delay in microseconds */
-int spin0sleep1;        /* 0=spin; 1=sleep */
-} CONC_DELAY_EVENT;
-
-/* Describe a barrier synchronization event. */
-typedef struct {
-MPI_Comm communicator;   /* Set of tasks to synchronize */
-} CONC_SYNC_EVENT;
-
-/* Describe a walk over a memory-region. */
-typedef struct {
-ncptl_int regionbytes;  /* Size in bytes of the region to touch */
-ncptl_int bytestride;   /* Stride in bytes to touch */
-ncptl_int numaccesses;  /* Number of words to touch */
-ncptl_int wordsize;     /* Size in bytes of each touch */
-ncptl_int firstbyte;    /* Byte offset of the first byte to touch */
-} CONC_TOUCH_EVENT;
-
-/* Describe a synchronous multicast event. */
-typedef struct {
-ncptl_int source;       /* Source task, -1 in the many-to-many case */
-ncptl_int size;         /* Number of bytes to send */
-ncptl_int alignment;    /* Message alignment (in bytes) */
-ncptl_int pendingsends; /* # of outstanding sends */
-ncptl_int pendingrecvs; /* # of outstanding receives */
-ncptl_int buffernum;    /* Buffer # to send/receive from */
-ncptl_int bufferofs;    /* Byte offset into the message buffer */
-ncptl_int tag;          /* Tag to use for selective receives */
-int misaligned;         /* 1=misaligned from a page; 0=align as specified */
-int touching;           /* 1=touch every word before sending */
-int verification;       /* 1=verify that all bits are correct */
-void *buffer;           /* Pointer to message memory */
-ncptl_int size2;   /* Number of bytes to receive in the many-to-many case */
-ncptl_int bufferofs2;   /* Byte offset into the message buffer in the many-to-many case */
-void * buffer2;   /* Pointer to receive-message memory in the many-to-many case */
-MPI_Comm communicator;   /* Set of tasks to multicast to/from */
-int root;   /* source's rank within communicator */
-int * sndvol;   /* Volume of data to send to each rank in the communicator */
-int * snddisp;   /* Offset from buffer of each message to send */
-int * rcvvol;   /* Volume of data to receive from each rank in the communicator */
-int * rcvdisp;   /* Offset from buffer2 of each message to receive */
-CONC_MCAST_MPI_FUNC mpi_func;   /* MPI function to use to perform the multicast */
-} CONC_MCAST_EVENT;
-
-/* Describe a reduction event. */
-typedef struct {
-ncptl_int numitems;     /* # of items to reduce */
-ncptl_int itemsize;     /* # of bytes per item */
-ncptl_int alignment;    /* Message alignment (in bytes) */
-ncptl_int pendingsends; /* # of outstanding sends */
-ncptl_int pendingrecvs; /* # of outstanding receives */
-ncptl_int buffernum;    /* Buffer # to send/receive from */
-ncptl_int bufferofs;    /* Byte offset into the message buffer */
-ncptl_int tag;          /* Tag to use for selective receives */
-int misaligned;         /* 1=misaligned from a page; 0=align as specified */
-int touching;           /* 1=touch every word before sending/after receiving */
-int sending;            /* 1=we're a sender */
-int receiving;          /* 1=we're a receiver */
-void *buffer;           /* Pointer to message memory */
-void * altbuffer;   /* Pointer to additional message memory */
-MPI_Comm sendcomm;   /* Set of tasks to reduce from */
-MPI_Comm recvcomm;   /* Set of tasks to reduce to */
-MPI_Datatype datatype;   /* MPI datatype to reduce */
-int reducetype;   /* 0=reduce; 1=allreduce; 2=reduce+bcast */
-ncptl_int reduceroot;   /* Root task of the reduction if reducetype is 0 or 2 */
-ncptl_int bcastroot;   /* Root task of the multicast if reducetype is 2 */
-} CONC_REDUCE_EVENT;
-
-/* Describe an event representing the beginning of a timed loop. */
-typedef struct {
-uint64_t usecs;         /* Requested loop duration */
-uint64_t warmup_usecs;  /* Requested duration of warmup loops */
-uint64_t starttime;     /* Time at which the loop state last changed */
-uint64_t itersleft;     /* # of iterations remaining */
-uint64_t previters;     /* # of iterations we performed last time */
-int prev_quiet;         /* Previous value of suppress_output */
-int timing_trial;       /* 1=performing a timing trial; 0=running for real */
-volatile int finished;  /* 1=time has expired; 0=still ticking */
-} CONC_BTIME_EVENT;
-
-/* Describe an event representing the end of a timed loop. */
-typedef struct {
-ncptl_int begin_event;  /* Index into eventlist[] of the corresponding BTIME event */
-} CONC_ETIME_EVENT;
-
-/* Describe an event representing repetitions of subsequent events. */
-typedef struct {
-ncptl_int end_event;    /* Index into eventlist[] of the last event to repeat */
-ncptl_int numreps;      /* # of repetitions to perform */
-} CONC_REPEAT_EVENT;
-
-/* Describe an event representing output suppression (either on or off). */
-typedef struct conc_suppress_event {
-int quiet;              /* 0=allow output; 1=suppress it */
-int prev_quiet;         /* Previous value of suppress_output */
-ncptl_int matching_event;  /* Event ID of the "suppression on" event */
-uint64_t stop_elapsed_usecs;   /* Time at which we suppressed output */
-} CONC_SUPPRESS_EVENT;
-
-/* Describe an event representing arbitrary code to execute at run time. */
-typedef struct {
-ncptl_int number;       /* Unique number corresponding to a specific piece of code */NCPTL_VIRT_PHYS_MAP *procmap;  /* Current mapping between tasks and processors */
-ncptl_int var_numwords;   /* Copy of var_numwords to use within a piece of code */
-} CONC_CODE_EVENT;
-
-/* Describe an arbitrary coNCePTuaL event. */
-typedef struct {
-CONC_EVENT_TYPE type;          /* Type of event */
-union {
-CONC_SEND_EVENT send;          /* Send state */
-CONC_RECV_EVENT recv;          /* Receive state */
-CONC_WAIT_EVENT wait;          /* Wait-for-completions state */
-CONC_DELAY_EVENT delay;        /* State for spins and sleeps */
-CONC_TOUCH_EVENT touch;        /* State for memory touching */
-CONC_SYNC_EVENT sync;          /* Synchronization state */
-CONC_MCAST_EVENT mcast;        /* Multicast state */
-CONC_REDUCE_EVENT reduce;      /* Reduction state */
-CONC_BTIME_EVENT btime;        /* Timed-loop state */
-CONC_ETIME_EVENT etime;        /* Additional timed-loop state */
-CONC_REPEAT_EVENT rep;         /* Repeated-events state */
-CONC_SUPPRESS_EVENT suppress;  /* State for suppressing output */
-CONC_CODE_EVENT code;          /* State for arbitrary code */
-} s;
-} CONC_EVENT;
-
-/* Fully specify an arbitrary for() loop (used by FOR EACH). */
-typedef struct {
-NCPTL_QUEUE *list_comp;  /* NULL=ordinary list; other=list comprehension values */
-int integral;        /* 1=integral values; 0=floating-point values */
-enum {               /* Comparison of loop variable to end variable */
-CONC_LEQ,                /* Increasing progression */
-CONC_GEQ                 /* Decreasing progression */
-} comparator;
-enum {               /* How to increment the loop variable */
-CONC_ADD,                /* Arithmetically */
-CONC_MULT,               /* Geometrically increasing */
-CONC_DIV                 /* Geometrically decreasing */
-} increment;
-union {
-struct {
-ncptl_int loopvar;   /* Loop variable */
-ncptl_int prev_loopvar; /* Previous value of loop variable */
-ncptl_int startval;  /* Initial value of loop variable */
-ncptl_int endval;    /* Value not to exceed */
-ncptl_int incval;    /* Loop-variable increment */
-} i;
-struct {
-double loopvar;      /* Loop variable */
-double prev_loopvar; /* Previous value of loop variable */
-double startval;     /* Initial value of loop variable */
-double endval;       /* Value not to exceed */
-double incval;       /* Loop-variable increment */
-} d;
-} u;
-} LOOPBOUNDS;
-
-/********************
- * Global variables *
- ********************/
-
-/* Variables exported to coNCePTuaL programs */
-static ncptl_int var_bytes_received = 0;   /* Total number of bytes received */
-static ncptl_int var_msgs_received = 0;   /* Total number of messages received */
-static ncptl_int var_bit_errors = 0;   /* Total number of bit errors observed */
-static ncptl_int var_total_msgs = 0;   /* Sum of messages sent and messages received */
-static ncptl_int var_msgs_sent = 0;   /* Total number of messages sent */
-static ncptl_int var_bytes_sent = 0;   /* Total number of bytes sent */
-static ncptl_int var_num_tasks = 1;   /* Number of tasks running the program */
-static ncptl_int var_elapsed_usecs = 0;   /* Elapsed time in microseconds */
-static ncptl_int var_total_bytes = 0;   /* Sum of bytes sent and bytes received */
-
-/* Dummy variable to help mark other variables as used */
-union {
-ncptl_int ni;
-int i;
-void *vp;
-} conc_dummy_var;
-
-/* Variables used internally by boilerplate code */
-static uint64_t starttime;   /* Time the clock was last reset (microseconds) */
-static ncptl_int pendingrecvs = 0;   /* Current # of outstanding receives */
-static ncptl_int pendingrecvbytes = 0; /* Current # of bytes in outstanding receives */
-static NCPTL_QUEUE *touchedqueue;      /* Queue of asynchronous receives to touch */
-static ncptl_int pendingsends = 0;   /* Current # of outstanding sends */
-static NCPTL_QUEUE *eventqueue;   /* List of coNCePTuaL events to perform */
-static int within_time_loop = 0;   /* 1=we're within a FOR <time> loop */
-static int suppress_output = 0;    /* 1=suppress output to stdout and the log file */
-static void *touch_region = NULL;   /* Memory region to touch */
-static ncptl_int touch_region_size = 0;   /* # of bytes in the above */
-static int virtrank;    /* This task's virtual rank in the computation */
-static int physrank;    /* This task's physical rank in the computation */
-static NCPTL_VIRT_PHYS_MAP *procmap;  /* Virtual to physical rank mapping */
-static NCPTL_LOG_FILE_STATE *logstate;   /* Opaque object representing all log-file state */
-static char *logfile_uuid;   /* Execution UUID to write to every log file */
-static char *logfiletmpl;   /* Template for the log file's name */
-static char *logfiletmpl_default;   /* Default value of the above */
-
-/* Global variables specific to the c_mpi backend */
-static ncptl_int mpi_is_running = 0;   /* 1=MPI has been initialized */
-static NCPTL_QUEUE * recvreqQ;   /* List of MPI receive requests */
-static MPI_Request * recvrequests;   /* List version of recvreqQ */
-static NCPTL_QUEUE * recvstatQ;   /* List of MPI receive statuses */
-static MPI_Status * recvstatuses;   /* List version of recvstatQ */
-static NCPTL_QUEUE * sendreqQ;   /* List of MPI send requests */
-static MPI_Request * sendrequests;   /* List version of sendreqQ */
-static NCPTL_QUEUE * sendstatQ;   /* List of MPI send statuses */
-static MPI_Status * sendstatuses;   /* List version of sendstatQ */
-static NCPTL_SET * communicators;   /* Map from an array of processor flags to an MPI communicator */
-static MPI_Errhandler mpi_error_handler;   /* Handle to handle_MPI_error() */
-static ncptl_int mpi_tag_ub;   /* Upper bound on an MPI tag value */
-static ncptl_int conc_mcast_tallies[CONC_MCAST_MPI_NUM_FUNCS] = {0};   /* Tallies of (static) multicast implementation functions */
-
-/* Program-specific variables */
-ncptl_int var_numwords;   /* Message size (words) (command-line argument) */
-ncptl_int var_reps;   /* Number of repetitions (command-line argument) */
-ncptl_int var_computetime;   /* Computation time (ms) (command-line argument) */
-
-/*************************
- * Function declarations *
- *************************/
-
-/* Make MPI errors invoke ncptl_fatal(). */
-static void handle_MPI_error (MPI_Comm *comm, int *errcode, ...)
-{
-va_list args;
-char errstring[MPI_MAX_ERROR_STRING];
-int errstrlen;
-
-va_start (args, errcode);
-if (MPI_Error_string (*errcode, errstring, &errstrlen) == MPI_SUCCESS)
-ncptl_fatal ("MPI run-time error: %s", errstring);
-else
-ncptl_fatal ("MPI aborted with unrecognized error code %d", *errcode);
-conc_dummy_var.vp = (void *) comm;   /* Prevent the compiler from complaining that comm is unused. */
-va_end (args);
-}
-
-/* Perform the equivalent of MPI_Comm_rank() for an arbitrary process. */
-static int rank_in_MPI_communicator (MPI_Comm subcomm, int global_rank)
-{
-  MPI_Group world_group;   /* Group associated with MPI_COMM_WORLD */
-  MPI_Group subgroup;      /* Group associate with subcomm */
-  int subrank;             /* global_rank's rank within subcomm */
-
-  MPI_Comm_group (MPI_COMM_WORLD, &world_group);
-  MPI_Comm_group (subcomm, &subgroup);
-  MPI_Group_translate_ranks (world_group, 1, &global_rank, subgroup, &subrank);
-  return subrank;
-}
-
-/* Map an arbitrary tag to within MPI's valid range of [0, mpi_tag_ub]. */
-static ncptl_int map_tag_into_MPI_range (ncptl_int tag)
-{
-if (tag == NCPTL_INT_MIN)
- /* Avoid taking the absolute value of NCPTL_INT_MIN. */
-tag = 555666773LL;   /* Arbitrary value */
-tag = ncptl_func_abs (tag);   /* Only nonnegatives values are allowed. */
-if (mpi_tag_ub < NCPTL_INT_MAX)
-tag %= mpi_tag_ub + 1;
-return tag;
-}
-
-/* Given an array of task in/out booleans return an MPI
- * communicator that represents the "in" tasks. */
-static MPI_Comm define_MPI_communicator (char *procflags)
-{
-MPI_Comm *existing_comm;    /* Previously defined MPI communicator */
-MPI_Comm new_comm;          /* Newly defined MPI communicator */
-
-existing_comm = (MPI_Comm *) ncptl_set_find (communicators, (void *)procflags);
-if (existing_comm)
-return *existing_comm;
-(void) MPI_Comm_split (MPI_COMM_WORLD, (int)procflags[physrank], physrank, &new_comm);
-(void) MPI_Errhandler_set (new_comm, mpi_error_handler);
-ncptl_set_insert (communicators, (void *)procflags, (void *)&new_comm);
-return define_MPI_communicator (procflags);
-}
-
-
-/* Inhibit the compiler from complaining that
- * certain variables are defined but not used.
- * This function should never be called. */
-void conc_mark_variables_used (void)
-{
-conc_dummy_var.ni = var_bytes_received;
-conc_dummy_var.ni = var_msgs_received;
-conc_dummy_var.ni = var_bit_errors;
-conc_dummy_var.ni = var_total_msgs;
-conc_dummy_var.ni = var_msgs_sent;
-conc_dummy_var.ni = var_bytes_sent;
-conc_dummy_var.ni = var_num_tasks;
-conc_dummy_var.ni = var_elapsed_usecs;
-conc_dummy_var.ni = var_total_bytes;
-conc_dummy_var.ni = pendingrecvbytes;
-conc_dummy_var.ni = touch_region_size;
-conc_dummy_var.vp = touch_region;
-conc_dummy_var.i  = within_time_loop;
-conc_dummy_var.i  = suppress_output;
-rank_in_MPI_communicator (MPI_COMM_WORLD, 0);
-}
-
-/* Allocate a new event of a given type and return a pointer to it. */
-static CONC_EVENT *conc_allocate_event (CONC_EVENT_TYPE type)
-{
-CONC_EVENT *newevent = (CONC_EVENT *) ncptl_queue_allocate (eventqueue);
-
-newevent->type = type;
-return newevent;
-}
-
-/* Declare an exit handler that gets called automatically when the
- * program terminates, whether successfully or not. */
-static void conc_exit_handler (void)
-{
-if (mpi_is_running)
-MPI_Abort (MPI_COMM_WORLD, 1);
-}
-
-
-/* Initialize coNCePTuaL, the messaging layer, and this program itself. */
-static void conc_initialize (int argc, char *argv[])
-{
- /* Variables needed by all C-based backends */
-CONC_EVENT * eventlist;   /* List of events to execute */
-ncptl_int numevents;   /* Number of entries in eventlist[] */
-int help_only = 0;   /* 1=User specified --help; save time by skipping ncptl_init() */
-char * argv0 = strrchr(argv[0], '/') ? strrchr(argv[0], '/')+1 : argv[0];   /* Base name of the executable program */
-int i;   /* Generic loop variable */
-
- /* Declare all of our command-line arguments. */
-NCPTL_CMDLINE arguments[] = {
-{ NCPTL_TYPE_STRING, NULL, "logfile", 'L', "Log-file template", {0}},
-{ NCPTL_TYPE_INT, NULL, "msgsize", 's', "Message size (words)", {0}},
-{ NCPTL_TYPE_INT, NULL, "reps", 'r', "Number of repetitions", {0}},
-{ NCPTL_TYPE_INT, NULL, "compute", 'c', "Computation time (ms)", {0}}
-};
-
- /* Incorporate the complete coNCePTuaL source code as an array
-  * for use by ncptl_log_write_prologue(). */
-char *sourcecode[] = {
-"# Measure the performance of MPI_Allgather()",
-"# By Scott Pakin <pakin@lanl.gov>",
-"#",
-"# N.B. Requires the c_mpi backend.",
-"Require language version \"1.5\".",
-"# Parse the command line.",
-"numwords is \"Message size (words)\" and comes from \"--msgsize\" or \"-s\" with default 28825K.",
-"reps is \"Number of repetitions\" and comes from \"--reps\" or \"-r\" with default 100.",
-"computetime is \"Computation time (ms)\" and comes from \"--compute\" or \"-c\" with default 129.",
-"",
-"# Allocate a send buffer and a receive buffer.",
-"Task 0 multicasts a numwords*num_tasks word message from buffer 0 to all other tasks.",
-"Task 0 multicasts a numwords*num_tasks word message from buffer 1 to all other tasks.",
-"",
-"# Measure the performance of MPI_Allreduce().",
-"Task 0 resets its counters then",
-"for reps repetitions {",
-"    all tasks COMPUTES FOR computetime MILLISECONDS then",
-"    all tasks backend execute \"",
-"        MPI_Allreduce([MESSAGE BUFFER 0], [MESSAGE BUFFER 1], (int)\" and numwords and \",",
-"                MPI_INT, MPI_SUM, MPI_COMM_WORLD);",
-"        \" then",
-"    all tasks backend execute \"",
-"        MPI_Allreduce([MESSAGE BUFFER 0], [MESSAGE BUFFER 1], (int)\" and numwords and \",",
-"                MPI_INT, MPI_SUM, MPI_COMM_WORLD);",
-"        \"",
-"} then",
-"task 0 logs elapsed_usecs/1000 as \"Elapse time (ms)\".",
-NULL
-};
-
- /* Variables specific to the c_mpi backend */
-int num_tasks;   /* int version of var_num_tasks needed by MPI_Comm_size() */
-char * procflags;   /* Array of 1s representing an all-task MPI communicator */
-MPI_Comm comm_world = MPI_COMM_WORLD;   /* Copy of MPI_COMM_WORLD that we can take the address of */
-void * attr_val;   /* Pointed to the value of MPI_TAG_UB */
-int attr_flag = 0;   /* true=MPI_TAG_UB was extracted; false=not extracted */
-char log_key_str[128];   /* String representing the range of valid MPI tags */
-
- /* As a special case, if the command line contains --help, then skip
-  * the coNCePTuaL initialization step. */
-for (i=1; i<argc; i++)
-if (!strcmp(argv[i], "--"))
-break;
-else
-if (!strcmp(argv[i], "--help") || !strcmp(argv[i], "-?")) {
-argv[1] = "-?";   /* Guaranteed to work, even with getopt() */
-help_only = 1;
-break;
-}
-
- /* Perform various initializations specific to the c_mpi backend. */
- /* Initialize MPI. */
-(void) MPI_Init(&argc, &argv);
-mpi_is_running = 1;
-
- /* Initialize the coNCePTuaL run-time library. */
-if (!help_only)
-ncptl_init (NCPTL_RUN_TIME_VERSION, argv[0]);
-(void) atexit (conc_exit_handler);
-
- /* Initialize the communication routines needed by the c_mpi backend. */
-(void) MPI_Errhandler_create ((MPI_Handler_function *)handle_MPI_error, &mpi_error_handler);
-(void) MPI_Errhandler_set (MPI_COMM_WORLD, mpi_error_handler);
-(void) MPI_Comm_rank(MPI_COMM_WORLD, &physrank);
-(void) MPI_Comm_size(MPI_COMM_WORLD, &num_tasks);
-var_num_tasks = (ncptl_int) num_tasks;
-(void) MPI_Comm_get_attr(MPI_COMM_WORLD, MPI_TAG_UB, &attr_val, &attr_flag);
-mpi_tag_ub = (ncptl_int) (attr_flag ? *(int *)attr_val : 32767);
-
- /* Generate and broadcast a UUID. */
-logfile_uuid = ncptl_log_generate_uuid();
-(void) MPI_Bcast ((void *)logfile_uuid, 37, MPI_CHAR, 0, MPI_COMM_WORLD);
-
- /* Plug variables and default values into the NCPTL_CMDLINE structure. */
-arguments[0].variable = (CMDLINE_VALUE *) &logfiletmpl;
-arguments[1].variable = (CMDLINE_VALUE *) &var_numwords;
-arguments[1].defaultvalue.intval = 29516800LL;
-arguments[2].variable = (CMDLINE_VALUE *) &var_reps;
-arguments[2].defaultvalue.intval = 100LL;
-arguments[3].variable = (CMDLINE_VALUE *) &var_computetime;
-arguments[3].defaultvalue.intval = 129LL;
-logfiletmpl_default = (char *) ncptl_malloc (strlen(argv0) + 15, 0);
-sprintf (logfiletmpl_default, "%s-%%p.log", argv0);
-arguments[0].defaultvalue.stringval = logfiletmpl_default;
-
- /* Parse the command line. */
-mpi_is_running = 0;   /* Don't invoke MPI_Abort() after --help. */
-ncptl_parse_command_line (argc, argv, arguments, sizeof(arguments)/sizeof(NCPTL_CMDLINE));
-if (help_only)
-ncptl_fatal ("Internal error in the c_generic backend: failed to exit after giving help");
-mpi_is_running = 1;
-
- /* Establish a mapping from (virtual) task IDs to (physical) ranks. */
-procmap = ncptl_allocate_task_map (var_num_tasks);
-virtrank = ncptl_physical_to_virtual (procmap, physrank);
-
- /* Perform initializations specific to the c_mpi backend. */
-ncptl_log_add_comment ("MPI send routines", "MPI_Send() and MPI_Isend()");
-ncptl_log_add_comment ("MPI reduction operation", REDUCE_OPERATION_NAME);
-sprintf (log_key_str, "[0, %" NICS "]", mpi_tag_ub);
-ncptl_log_add_comment ("MPI tag range", log_key_str);
-
- /* Open the log file and write some standard prologue information to it. */
-logstate = ncptl_log_open (logfiletmpl, physrank);
-ncptl_log_write_prologue (logstate, argv[0], logfile_uuid, "c_mpi", "C + MPI",
-var_num_tasks,
-arguments, sizeof(arguments)/sizeof(NCPTL_CMDLINE),
-sourcecode);
-ncptl_free (logfile_uuid);
-
- /* Allocate a variety of dynamically growing queues. */
-eventqueue = ncptl_queue_init (sizeof (CONC_EVENT));
-touchedqueue = ncptl_queue_init (sizeof (ncptl_int));
-
- /* Perform initializations specific to the c_mpi backend. */
-sendreqQ = ncptl_queue_init (sizeof (MPI_Request));
-sendstatQ = ncptl_queue_init (sizeof (MPI_Status));
-recvreqQ = ncptl_queue_init (sizeof (MPI_Request));
-recvstatQ = ncptl_queue_init (sizeof (MPI_Status));
-communicators = ncptl_set_init (ESTIMATED_COMMUNICATORS, var_num_tasks*sizeof(char), sizeof(MPI_Comm));procflags = (char *) ncptl_malloc (var_num_tasks*sizeof(char), 0);
-for (i=0; i<var_num_tasks; i++)
-procflags[i] = 1;
-ncptl_set_insert (communicators, (void *)procflags, (void *)&comm_world);
-ncptl_free (procflags);
-
- /****************************************************
-  * Generated, program-specific initialization code. *
-  ****************************************************/
-
- /* TASK 0LL MULTICAST...TO ALL OTHER TASKS */
-{
-ncptl_int numsenders = 0LL;   /* Number of sending tasks */
-int * sndvol;   /* Number of bytes we send to each other task */
-int * snddisp;   /* Buffer offset of each send */
-int sndnum = 0;   /* Total number of sends from us */
-int * rcvvol;   /* Number of bytes sent to us from each other task */
-int * rcvdisp;   /* Buffer offset of each receive */
-int rcvnum = 0;   /* Total number of sends to us */
-int peervar;   /* Physical rank of one of our peer tasks */
-char * procflags = (char *) ncptl_malloc (var_num_tasks*sizeof(char), 0);   /* Flags indicating whether each task is in or out */
-ncptl_int * sendsfrom = (ncptl_int *) ncptl_malloc (var_num_tasks*sizeof(ncptl_int), 0);   /* Tally of sends from each rank */
-ncptl_int * recvsby = (ncptl_int *) ncptl_malloc (var_num_tasks*sizeof(ncptl_int), 0);   /* Tally of receives by each rank */
-int stasknum;   /* A single source task mapped by ncptl_virtual_to_physical() */
-int ttasknum;   /* A single target task mapped by ncptl_virtual_to_physical() */
-CONC_MCAST_MPI_FUNC mpi_func;   /* The MPI function that will implement the multicast */
-MPI_Comm subcomm;   /* MPI subcommunicator to use */
-
- /* Determine all participants in the many-to-many multicast. */
-memset(procflags, 0, var_num_tasks);
-memset(sendsfrom, 0, var_num_tasks*sizeof(ncptl_int));
-memset(recvsby, 0, var_num_tasks*sizeof(ncptl_int));
-if ((0LL) >= 0 && (0LL) < var_num_tasks) {
-stasknum = (int) ncptl_virtual_to_physical(procmap, 0LL);
-procflags[stasknum] = 1;
-numsenders = 1;
-{
-ncptl_int ivar_a;
-
-for (ivar_a=0; ivar_a<var_num_tasks; ivar_a++)
-if (ivar_a != 0LL) {
-ttasknum = (int) ncptl_virtual_to_physical(procmap, ivar_a);
-sendsfrom[stasknum]++;
-recvsby[ttasknum]++;
-procflags[ttasknum] = 1;
-}
-}
-}
-subcomm = define_MPI_communicator(procflags);
-
- /* Determine if all participants are sending and receiving the same number
-  * and volume of messages.  If so, then we can use the faster MPI_Alltoall()
-  * function for the multicast instead of the slower MPI_Alltoallv(). */
-{
-ncptl_int msgtally = -1;   /* Messages sent or received by any participant */
-ncptl_int i;
-mpi_func = CONC_MCAST_MPI_ALLTOALL;  /* Use MPI_Alltoall() unless we require MPI_Alltoallv(). */
-for (i=0; i<var_num_tasks; i++) {
-stasknum = (int) ncptl_virtual_to_physical(procmap, i);
-if (procflags[stasknum]) {
-if (msgtally == -1)
-msgtally = sendsfrom[stasknum];
-if (sendsfrom[stasknum] != msgtally || recvsby[stasknum] != msgtally) {
-mpi_func = CONC_MCAST_MPI_ALLTOALLV;
-break;
-}
-}
-}
-}
-
- /* The following steps are performed only by those tasks who are
-  * involved in the communication (as senders and/or receivers). */
-if (numsenders > 0 && procflags[physrank]) {
-int groupsize = 0;   /* Number of MPI ranks represented by subcomm */
-ncptl_int ivar_b_loop;
-
-if (numsenders == 1) {
- /* As a special case, use MPI_Bcast() if there's a single sender. */
-conc_mcast_tallies[CONC_MCAST_MPI_BCAST]++;
-CONC_EVENT *thisev = conc_allocate_event (EV_MCAST);
-thisev->s.mcast.source = ncptl_virtual_to_physical (procmap, 0LL);
-thisev->s.mcast.size = ((var_numwords)*(var_num_tasks))*4LL;
-thisev->s.mcast.alignment = 0LL;
-thisev->s.mcast.misaligned = 0;
-thisev->s.mcast.touching = 0;
-thisev->s.mcast.verification = 0;
-thisev->s.mcast.tag = 0LL;
-thisev->s.mcast.pendingsends = pendingsends;
-thisev->s.mcast.pendingrecvs = pendingrecvs;
-thisev->s.mcast.bufferofs = 0LL;
-if (0LL < 0LL)
-thisev->s.mcast.buffernum = thisev->s.mcast.verification ? pendingsends+pendingrecvs : 0;
-else
-thisev->s.mcast.buffernum = 0LL;
-(void) ncptl_malloc_message (thisev->s.mcast.size+thisev->s.mcast.bufferofs, thisev->s.mcast.alignment, thisev->s.mcast.buffernum, thisev->s.mcast.misaligned);
-thisev->s.mcast.buffer = NULL;
-if (var_num_tasks == 1)
-thisev->s.mcast.source = 0LL;
-thisev->s.mcast.mpi_func = CONC_MCAST_MPI_BCAST;  /* One-to-many communication */
-
-thisev->s.mcast.communicator = subcomm;
-thisev->s.mcast.root = rank_in_MPI_communicator (thisev->s.mcast.communicator, thisev->s.mcast.source);
-if (thisev->s.mcast.tag != 0LL)
-ncptl_fatal ("The c_mpi backend does not support nonzero tags in MULTICAST statements");
-}
-else {
- /* We have more than one sender.
-  * Allocate memory for MPI_Alltoallv() to use. */
-for (ivar_b_loop=0; ivar_b_loop<var_num_tasks; ivar_b_loop++)
-groupsize += procflags[ivar_b_loop];
-sndvol = (int *) ncptl_malloc(groupsize*sizeof(int), 0);
-memset(sndvol, 0, groupsize*sizeof(int));
-snddisp = (int *) ncptl_malloc(groupsize*sizeof(int), 0);
-memset(snddisp, 0, groupsize*sizeof(int));
-rcvvol = (int *) ncptl_malloc(groupsize*sizeof(int), 0);
-memset(rcvvol, 0, groupsize*sizeof(int));
-rcvdisp = (int *) ncptl_malloc(groupsize*sizeof(int), 0);
-memset(rcvdisp, 0, groupsize*sizeof(int));
-
- /* Determine each task to which I will send data. */
-if ((0LL) == virtrank) {   /* TASK 0LL */
- /* The current coNCePTuaL statement applies to our task. */
-ncptl_int ivar_c_loop;
-for (ivar_c_loop=0; ivar_c_loop<var_num_tasks; ivar_c_loop++)
-if (ivar_c_loop != virtrank) {
- /* In this scope, ivar_c_loop represents a single receiver. */
-peervar = rank_in_MPI_communicator (subcomm, ncptl_virtual_to_physical(procmap, ivar_c_loop));
-sndvol[peervar] = (int) (((var_numwords)*(var_num_tasks))*4LL);
-snddisp[peervar] = (int) (sndnum * (((var_numwords)*(var_num_tasks))*4LL));
-sndnum++;
-}
-}
-
- /* Determine each task who will send data to me. */
-if ((0LL)>=0 && (0LL)<var_num_tasks) {
- /* 0LL now represents one of the tasks that will send to us. */
-ncptl_int ivar_d_task = 0LL;
-if (virtrank != 0LL) {
- /* Register that task ivar_d_task will send us data. */
-peervar = rank_in_MPI_communicator (subcomm, ncptl_virtual_to_physical(procmap, ivar_d_task));
-rcvvol[peervar] = (int) (((var_numwords)*(var_num_tasks))*4LL);
-rcvdisp[peervar] = (int) (rcvnum * (((var_numwords)*(var_num_tasks))*4LL));
-rcvnum++;
-}
-}
-
- /* MPI_Alltoall() includes sends to self.  Adjust sndnum and rcvnum appropriately. */
-if (mpi_func == CONC_MCAST_MPI_ALLTOALL) {
-MPI_Comm_size (subcomm, &sndnum);
-rcvnum = sndnum;
-}
-
- /* Prepare to multicast a message. */
-conc_mcast_tallies[mpi_func]++;
-if (1) {
-CONC_EVENT *thisev = conc_allocate_event (EV_MCAST);
-thisev->s.mcast.size = sndnum * (((var_numwords)*(var_num_tasks))*4LL);
-thisev->s.mcast.alignment = 0LL;
-thisev->s.mcast.misaligned = 0;
-thisev->s.mcast.touching = 0;
-thisev->s.mcast.verification = 0;
-thisev->s.mcast.tag = 0LL;
-thisev->s.mcast.pendingsends = pendingsends;
-thisev->s.mcast.pendingrecvs = pendingrecvs;
-thisev->s.mcast.bufferofs = 0LL;
-if (0LL < 0LL)
-thisev->s.mcast.buffernum = thisev->s.mcast.verification ? pendingsends+pendingrecvs : 0;
-else
-thisev->s.mcast.buffernum = 0LL;
-(void) ncptl_malloc_message (thisev->s.mcast.size+thisev->s.mcast.bufferofs, thisev->s.mcast.alignment, thisev->s.mcast.buffernum, thisev->s.mcast.misaligned);
-thisev->s.mcast.buffer = NULL;
-thisev->s.mcast.bufferofs2 = 0LL;
-thisev->s.mcast.size2 = rcvnum * (((var_numwords)*(var_num_tasks))*4LL);
-(void) ncptl_malloc_message (thisev->s.mcast.size2 + thisev->s.mcast.bufferofs2, thisev->s.mcast.alignment, thisev->s.mcast.buffernum+1, thisev->s.mcast.misaligned);
-thisev->s.mcast.buffer2 = NULL;
-thisev->s.mcast.communicator = subcomm;
-if (thisev->s.mcast.tag != 0LL)
-ncptl_fatal ("The c_mpi backend does not support nonzero tags in MULTICAST statements");
-thisev->s.mcast.sndvol = sndvol;
-thisev->s.mcast.snddisp = snddisp;
-thisev->s.mcast.rcvvol = rcvvol;
-thisev->s.mcast.rcvdisp = rcvdisp;
-thisev->s.mcast.source = -1;
-thisev->s.mcast.mpi_func = mpi_func;
-if (mpi_func == CONC_MCAST_MPI_ALLTOALL)
-thisev->s.mcast.sndvol[0] = thisev->s.mcast.rcvvol[0] = ((var_numwords)*(var_num_tasks))*4LL;
-}
-}
-}
-ncptl_free(procflags);
-ncptl_free(sendsfrom);
-ncptl_free(recvsby);
-}
- /* TASK 0LL MULTICAST...TO ALL OTHER TASKS */
-{
-ncptl_int numsenders = 0LL;   /* Number of sending tasks */
-int * sndvol;   /* Number of bytes we send to each other task */
-int * snddisp;   /* Buffer offset of each send */
-int sndnum = 0;   /* Total number of sends from us */
-int * rcvvol;   /* Number of bytes sent to us from each other task */
-int * rcvdisp;   /* Buffer offset of each receive */
-int rcvnum = 0;   /* Total number of sends to us */
-int peervar;   /* Physical rank of one of our peer tasks */
-char * procflags = (char *) ncptl_malloc (var_num_tasks*sizeof(char), 0);   /* Flags indicating whether each task is in or out */
-ncptl_int * sendsfrom = (ncptl_int *) ncptl_malloc (var_num_tasks*sizeof(ncptl_int), 0);   /* Tally of sends from each rank */
-ncptl_int * recvsby = (ncptl_int *) ncptl_malloc (var_num_tasks*sizeof(ncptl_int), 0);   /* Tally of receives by each rank */
-int stasknum;   /* A single source task mapped by ncptl_virtual_to_physical() */
-int ttasknum;   /* A single target task mapped by ncptl_virtual_to_physical() */
-CONC_MCAST_MPI_FUNC mpi_func;   /* The MPI function that will implement the multicast */
-MPI_Comm subcomm;   /* MPI subcommunicator to use */
-
- /* Determine all participants in the many-to-many multicast. */
-memset(procflags, 0, var_num_tasks);
-memset(sendsfrom, 0, var_num_tasks*sizeof(ncptl_int));
-memset(recvsby, 0, var_num_tasks*sizeof(ncptl_int));
-if ((0LL) >= 0 && (0LL) < var_num_tasks) {
-stasknum = (int) ncptl_virtual_to_physical(procmap, 0LL);
-procflags[stasknum] = 1;
-numsenders = 1;
-{
-ncptl_int ivar_e;
-
-for (ivar_e=0; ivar_e<var_num_tasks; ivar_e++)
-if (ivar_e != 0LL) {
-ttasknum = (int) ncptl_virtual_to_physical(procmap, ivar_e);
-sendsfrom[stasknum]++;
-recvsby[ttasknum]++;
-procflags[ttasknum] = 1;
-}
-}
-}
-subcomm = define_MPI_communicator(procflags);
-
- /* Determine if all participants are sending and receiving the same number
-  * and volume of messages.  If so, then we can use the faster MPI_Alltoall()
-  * function for the multicast instead of the slower MPI_Alltoallv(). */
-{
-ncptl_int msgtally = -1;   /* Messages sent or received by any participant */
-ncptl_int i;
-mpi_func = CONC_MCAST_MPI_ALLTOALL;  /* Use MPI_Alltoall() unless we require MPI_Alltoallv(). */
-for (i=0; i<var_num_tasks; i++) {
-stasknum = (int) ncptl_virtual_to_physical(procmap, i);
-if (procflags[stasknum]) {
-if (msgtally == -1)
-msgtally = sendsfrom[stasknum];
-if (sendsfrom[stasknum] != msgtally || recvsby[stasknum] != msgtally) {
-mpi_func = CONC_MCAST_MPI_ALLTOALLV;
-break;
-}
-}
-}
-}
-
- /* The following steps are performed only by those tasks who are
-  * involved in the communication (as senders and/or receivers). */
-if (numsenders > 0 && procflags[physrank]) {
-int groupsize = 0;   /* Number of MPI ranks represented by subcomm */
-ncptl_int ivar_f_loop;
-
-if (numsenders == 1) {
- /* As a special case, use MPI_Bcast() if there's a single sender. */
-conc_mcast_tallies[CONC_MCAST_MPI_BCAST]++;
-CONC_EVENT *thisev = conc_allocate_event (EV_MCAST);
-thisev->s.mcast.source = ncptl_virtual_to_physical (procmap, 0LL);
-thisev->s.mcast.size = ((var_numwords)*(var_num_tasks))*4LL;
-thisev->s.mcast.alignment = 0LL;
-thisev->s.mcast.misaligned = 0;
-thisev->s.mcast.touching = 0;
-thisev->s.mcast.verification = 0;
-thisev->s.mcast.tag = 0LL;
-thisev->s.mcast.pendingsends = pendingsends;
-thisev->s.mcast.pendingrecvs = pendingrecvs;
-thisev->s.mcast.bufferofs = 0LL;
-if (1LL < 0LL)
-thisev->s.mcast.buffernum = thisev->s.mcast.verification ? pendingsends+pendingrecvs : 0;
-else
-thisev->s.mcast.buffernum = 1LL;
-(void) ncptl_malloc_message (thisev->s.mcast.size+thisev->s.mcast.bufferofs, thisev->s.mcast.alignment, thisev->s.mcast.buffernum, thisev->s.mcast.misaligned);
-thisev->s.mcast.buffer = NULL;
-if (var_num_tasks == 1)
-thisev->s.mcast.source = 0LL;
-thisev->s.mcast.mpi_func = CONC_MCAST_MPI_BCAST;  /* One-to-many communication */
-
-thisev->s.mcast.communicator = subcomm;
-thisev->s.mcast.root = rank_in_MPI_communicator (thisev->s.mcast.communicator, thisev->s.mcast.source);
-if (thisev->s.mcast.tag != 0LL)
-ncptl_fatal ("The c_mpi backend does not support nonzero tags in MULTICAST statements");
-}
-else {
- /* We have more than one sender.
-  * Allocate memory for MPI_Alltoallv() to use. */
-for (ivar_f_loop=0; ivar_f_loop<var_num_tasks; ivar_f_loop++)
-groupsize += procflags[ivar_f_loop];
-sndvol = (int *) ncptl_malloc(groupsize*sizeof(int), 0);
-memset(sndvol, 0, groupsize*sizeof(int));
-snddisp = (int *) ncptl_malloc(groupsize*sizeof(int), 0);
-memset(snddisp, 0, groupsize*sizeof(int));
-rcvvol = (int *) ncptl_malloc(groupsize*sizeof(int), 0);
-memset(rcvvol, 0, groupsize*sizeof(int));
-rcvdisp = (int *) ncptl_malloc(groupsize*sizeof(int), 0);
-memset(rcvdisp, 0, groupsize*sizeof(int));
-
- /* Determine each task to which I will send data. */
-if ((0LL) == virtrank) {   /* TASK 0LL */
- /* The current coNCePTuaL statement applies to our task. */
-ncptl_int ivar_g_loop;
-for (ivar_g_loop=0; ivar_g_loop<var_num_tasks; ivar_g_loop++)
-if (ivar_g_loop != virtrank) {
- /* In this scope, ivar_g_loop represents a single receiver. */
-peervar = rank_in_MPI_communicator (subcomm, ncptl_virtual_to_physical(procmap, ivar_g_loop));
-sndvol[peervar] = (int) (((var_numwords)*(var_num_tasks))*4LL);
-snddisp[peervar] = (int) (sndnum * (((var_numwords)*(var_num_tasks))*4LL));
-sndnum++;
-}
-}
-
- /* Determine each task who will send data to me. */
-if ((0LL)>=0 && (0LL)<var_num_tasks) {
- /* 0LL now represents one of the tasks that will send to us. */
-ncptl_int ivar_h_task = 0LL;
-if (virtrank != 0LL) {
- /* Register that task ivar_h_task will send us data. */
-peervar = rank_in_MPI_communicator (subcomm, ncptl_virtual_to_physical(procmap, ivar_h_task));
-rcvvol[peervar] = (int) (((var_numwords)*(var_num_tasks))*4LL);
-rcvdisp[peervar] = (int) (rcvnum * (((var_numwords)*(var_num_tasks))*4LL));
-rcvnum++;
-}
-}
-
- /* MPI_Alltoall() includes sends to self.  Adjust sndnum and rcvnum appropriately. */
-if (mpi_func == CONC_MCAST_MPI_ALLTOALL) {
-MPI_Comm_size (subcomm, &sndnum);
-rcvnum = sndnum;
-}
-
- /* Prepare to multicast a message. */
-conc_mcast_tallies[mpi_func]++;
-if (1) {
-CONC_EVENT *thisev = conc_allocate_event (EV_MCAST);
-thisev->s.mcast.size = sndnum * (((var_numwords)*(var_num_tasks))*4LL);
-thisev->s.mcast.alignment = 0LL;
-thisev->s.mcast.misaligned = 0;
-thisev->s.mcast.touching = 0;
-thisev->s.mcast.verification = 0;
-thisev->s.mcast.tag = 0LL;
-thisev->s.mcast.pendingsends = pendingsends;
-thisev->s.mcast.pendingrecvs = pendingrecvs;
-thisev->s.mcast.bufferofs = 0LL;
-if (1LL < 0LL)
-thisev->s.mcast.buffernum = thisev->s.mcast.verification ? pendingsends+pendingrecvs : 0;
-else
-thisev->s.mcast.buffernum = 1LL;
-(void) ncptl_malloc_message (thisev->s.mcast.size+thisev->s.mcast.bufferofs, thisev->s.mcast.alignment, thisev->s.mcast.buffernum, thisev->s.mcast.misaligned);
-thisev->s.mcast.buffer = NULL;
-thisev->s.mcast.bufferofs2 = 0LL;
-thisev->s.mcast.size2 = rcvnum * (((var_numwords)*(var_num_tasks))*4LL);
-(void) ncptl_malloc_message (thisev->s.mcast.size2 + thisev->s.mcast.bufferofs2, thisev->s.mcast.alignment, thisev->s.mcast.buffernum+1, thisev->s.mcast.misaligned);
-thisev->s.mcast.buffer2 = NULL;
-thisev->s.mcast.communicator = subcomm;
-if (thisev->s.mcast.tag != 0LL)
-ncptl_fatal ("The c_mpi backend does not support nonzero tags in MULTICAST statements");
-thisev->s.mcast.sndvol = sndvol;
-thisev->s.mcast.snddisp = snddisp;
-thisev->s.mcast.rcvvol = rcvvol;
-thisev->s.mcast.rcvdisp = rcvdisp;
-thisev->s.mcast.source = -1;
-thisev->s.mcast.mpi_func = mpi_func;
-if (mpi_func == CONC_MCAST_MPI_ALLTOALL)
-thisev->s.mcast.sndvol[0] = thisev->s.mcast.rcvvol[0] = ((var_numwords)*(var_num_tasks))*4LL;
-}
-}
-}
-ncptl_free(procflags);
-ncptl_free(sendsfrom);
-ncptl_free(recvsby);
-}
- /* TASK 0LL RESET THEIR COUNTERS */
-if ((0LL) == virtrank) {   /* TASK 0LL */
- /* The current coNCePTuaL statement applies to our task. */
-(void) conc_allocate_event (EV_RESET);
-}
- /* THEN... */
- /* FOR var_reps REPETITIONS... */
-{
-ncptl_int numreps = var_reps;   /* Total # of repetitions to perform */
-int unroll_loop = numreps <= CONC_MAX_UNROLL;   /* 1=unroll loop; 0=use a REPEAT event */
-ncptl_int repnum;   /* Current repetition number */
-CONC_EVENT * repevent = NULL;   /* Event designating repetition */
-ncptl_int repeventnum = -1LL;   /* Event number corresponding to repevent */
- /* Conditionally unroll the loop. */
-for (repnum=0; repnum<(unroll_loop?numreps:1); repnum++) {
- /* Allocate a repeat event if we have more than one repetition. */
-if (!unroll_loop && numreps > 1LL) {
-repeventnum = ncptl_queue_length (eventqueue);
-repevent =  conc_allocate_event (EV_REPEAT);
-repevent->s.rep.numreps = numreps;
-}
-
- /* Output a loop body if we have at least one repetition. */
-if (unroll_loop || numreps > 0LL) {
- /* ALL TASKS COMPUTES FOR (var_computetime)*(1000LL) MICROSECONDS */
-{   /* ALL TASKS */
- /* The current coNCePTuaL statement applies to our task. */
-CONC_EVENT *thisev = conc_allocate_event (EV_DELAY);
-thisev->s.delay.microseconds = (uint64_t) var_computetime;
-thisev->s.delay.microseconds *= (uint64_t) 1000LL;
-thisev->s.delay.spin0sleep1 = 0;
-}
- /* THEN... */
- /* ALL TASKS BACKEND EXECUTES ['"\\n        MPI_Allreduce([MESSAGE BUFFER 0], [MESSAGE BUFFER 1], (int)"', 'var_numwords', '",\\n                MPI_INT, MPI_SUM, MPI_COMM_WORLD);\\n        "'] */
-{   /* ALL TASKS */
- /* The current coNCePTuaL statement applies to our task. */
-CONC_EVENT *thisev = conc_allocate_event (EV_CODE);
-thisev->s.code.number = 0;
-thisev->s.code.procmap = NULL;
-thisev->s.code.var_numwords = var_numwords;
-}
- /* THEN... */
- /* ALL TASKS BACKEND EXECUTES ['"\\n        MPI_Allreduce([MESSAGE BUFFER 0], [MESSAGE BUFFER 1], (int)"', 'var_numwords', '",\\n                MPI_INT, MPI_SUM, MPI_COMM_WORLD);\\n        "'] */
-{   /* ALL TASKS */
- /* The current coNCePTuaL statement applies to our task. */
-CONC_EVENT *thisev = conc_allocate_event (EV_CODE);
-thisev->s.code.number = 1;
-thisev->s.code.procmap = NULL;
-thisev->s.code.var_numwords = var_numwords;
-}
-}
-}
-
- /* Assign the number of events to repeat, now that we know that number. */
-if (!unroll_loop && numreps > 1LL) {
-repevent = repeventnum + (CONC_EVENT *) ncptl_queue_contents (eventqueue, 0);
-repevent->s.rep.end_event = ncptl_queue_length (eventqueue) - 1;
-}
-}
- /* THEN... */
- /* TASK 0LL LOGS "Elapse time (ms)" */
-if ((0LL) == virtrank) {   /* TASK 0LL */
- /* The current coNCePTuaL statement applies to our task. */
-CONC_EVENT *thisev = conc_allocate_event (EV_CODE);
-thisev->s.code.number = 2;
-thisev->s.code.procmap = NULL;
-}
- /* Begin a new top-level statement. */
-(void) conc_allocate_event (EV_NEWSTMT);
-
- /*************************
-  * More boilerplate code *
-  *************************/
-
- /* Abort if the program will terminate with pending messages. */
-if (pendingsends && pendingrecvs)
-ncptl_fatal("Neglected to await the completion of %" NICS " asynchronous %s and %" NICS " asynchronous %s",
-pendingsends, pendingsends==1LL ? "send" : "sends",
-pendingrecvs, pendingrecvs==1LL ? "receive" : "receives");
-else
-if (pendingsends)
-ncptl_fatal("Neglected to await the completion of %" NICS " asynchronous %s",
-pendingsends, pendingsends==1LL ? "send" : "sends");
-else
-if (pendingrecvs)
-ncptl_fatal("Neglected to await the completion of %" NICS " asynchronous %s",
-pendingrecvs, pendingrecvs==1LL ? "receive" : "receives");
-
- /* Allocate memory for non-unique messages and asynchronous
-  * message handles now that we know how much memory we need
-  * to allocate. */
-eventlist = (CONC_EVENT *) ncptl_queue_contents (eventqueue, 0);
-numevents = ncptl_queue_length (eventqueue);
-sendrequests = (MPI_Request *) ncptl_queue_contents (sendreqQ, 0);
-recvrequests = (MPI_Request *) ncptl_queue_contents (recvreqQ, 0);
-for (i=0; i<numevents; i++) {
-CONC_EVENT *thisev = &eventlist[i];   /* Cache of the current event */
-switch (thisev->type) {
-case EV_MCAST:
-if (!thisev->s.mcast.buffer)
-thisev->s.mcast.buffer = ncptl_malloc_message (thisev->s.mcast.bufferofs + thisev->s.mcast.size,
-thisev->s.mcast.alignment,
-thisev->s.mcast.buffernum,
-thisev->s.mcast.misaligned);
-if (thisev->s.mcast.verification)
-ncptl_fill_buffer (CONC_GETBUFPTR(mcast), thisev->s.mcast.bufferofs + thisev->s.mcast.size, -1);
-if (thisev->s.mcast.mpi_func == CONC_MCAST_MPI_ALLTOALL || thisev->s.mcast.mpi_func == CONC_MCAST_MPI_ALLTOALLV)
-thisev->s.mcast.buffer2 = ncptl_malloc_message (thisev->s.mcast.bufferofs2 + thisev->s.mcast.size2,
-thisev->s.mcast.alignment,
-thisev->s.mcast.buffernum + 1,
-thisev->s.mcast.misaligned);
-break;
-
-default:
-break;
-}
-}
-}
-
-/* Process a subset of the events in a given event list. */
-static void conc_process_events (CONC_EVENT *eventlist,
-ncptl_int firstev, ncptl_int lastev, ncptl_int numreps)
-{
-CONC_EVENT * thisev;   /* Cache of the current event */
-CONC_EVENT * thisev_first = &eventlist[firstev];   /* Cache of the first event */
-ncptl_int i;   /* Iterate over events. */
-ncptl_int j;   /* Iterate over repetitions. */
-
- /* Process from event firstev to event lastev (both inclusive). */
-for (j=numreps; j>0; j--)
-for (i=firstev, thisev=thisev_first; i<=lastev; i++, thisev++) {
- /* Declare variables needed by all C-based backends. */
-
- /* Process a single event. */
-switch (thisev->type) {
-case EV_MCAST:
- /* Synchronous multicast */
-switch (thisev->s.mcast.mpi_func) {
-case CONC_MCAST_MPI_BCAST:
- /* One to many */
-(void) MPI_Bcast (CONC_GETBUFPTR(mcast), thisev->s.mcast.size, MPI_BYTE,
-thisev->s.mcast.root, thisev->s.mcast.communicator);
-break;
-case CONC_MCAST_MPI_ALLTOALL:
- /* Many to many, same to each */
-(void) MPI_Alltoall (CONC_GETBUFPTR(mcast), thisev->s.mcast.sndvol[0], MPI_BYTE,
-(void *)((char *)thisev->s.mcast.buffer2 + thisev->s.mcast.bufferofs2), thisev->s.mcast.rcvvol[0], MPI_BYTE, thisev->s.mcast.communicator);
-break;
-case CONC_MCAST_MPI_ALLTOALLV:
- /* Many to many, different to each */
-(void) MPI_Alltoallv (CONC_GETBUFPTR(mcast), thisev->s.mcast.sndvol, thisev->s.mcast.snddisp, MPI_BYTE,
-(void *)((char *)thisev->s.mcast.buffer2 + thisev->s.mcast.bufferofs2), thisev->s.mcast.rcvvol, thisev->s.mcast.rcvdisp, MPI_BYTE, thisev->s.mcast.communicator);
-break;
-default:
- /* We should never get here. */
-ncptl_fatal ("Internal error: Unrecognized multicast function");
-break;
-}
-break;
-
-case EV_DELAY:
- /* Do nothing for a prescribed length of time. */
-ncptl_udelay (thisev->s.delay.microseconds, thisev->s.delay.spin0sleep1);
-break;
-
-case EV_RESET:
- /* Reset all of the counters exported to coNCePTuaL programs. */
-starttime = ncptl_time();
-break;
-
-case EV_REPEAT:
- /* Repeatedly perform the next batch of events. */
-conc_process_events (eventlist, i+1, thisev->s.rep.end_event, thisev->s.rep.numreps);
-i = thisev->s.rep.end_event;
-thisev = &eventlist[i];
-break;
-
-case EV_NEWSTMT:
- /* Begin a new table in the log file. */
-if (!suppress_output) {
-uint64_t stop_elapsed_usecs = ncptl_time();
-var_elapsed_usecs = stop_elapsed_usecs - starttime;
-ncptl_log_commit_data (logstate);
-starttime += ncptl_time() - stop_elapsed_usecs;
-}
-break;
-
-case EV_CODE:
- /* Execute an arbitrary piece of code. */
-switch (thisev->s.code.number) {
-case 0:
- /* ALL TASKS BACKEND EXECUTES ['"\\n        MPI_Allreduce([MESSAGE BUFFER 0], [MESSAGE BUFFER 1], (int)"', 'var_numwords', '",\\n                MPI_INT, MPI_SUM, MPI_COMM_WORLD);\\n        "'] */
-{
-
-        MPI_Allreduce((ncptl_get_message_buffer ((ncptl_int)(0))), (ncptl_get_message_buffer ((ncptl_int)(1))), (int) (double)thisev->s.code.var_numwords ,
-                MPI_INT, MPI_SUM, MPI_COMM_WORLD);
-        
-}
-break;
-
-case 1:
- /* ALL TASKS BACKEND EXECUTES ['"\\n        MPI_Allreduce([MESSAGE BUFFER 0], [MESSAGE BUFFER 1], (int)"', 'var_numwords', '",\\n                MPI_INT, MPI_SUM, MPI_COMM_WORLD);\\n        "'] */
-{
-
-        MPI_Allreduce((ncptl_get_message_buffer ((ncptl_int)(0))), (ncptl_get_message_buffer ((ncptl_int)(1))), (int) (double)thisev->s.code.var_numwords ,
-                MPI_INT, MPI_SUM, MPI_COMM_WORLD);
-        
-}
-break;
-
-case 2:
- /* TASK 0LL LOGS "Elapse time (ms)" */
-if (!suppress_output) {
-uint64_t stop_elapsed_usecs = ncptl_time();
-var_elapsed_usecs = stop_elapsed_usecs - starttime;
-ncptl_log_write (logstate, 0, "Elapse time (ms)", NCPTL_FUNC_NO_AGGREGATE, 0.0, ((double)var_elapsed_usecs)/(1000.0));
-starttime += ncptl_time() - stop_elapsed_usecs;
-}
-break;
-
-default:
- /* The C code generation module must be broken. */
-ncptl_fatal ("Internal error: unknown EV_CODE block %" NICS, thisev->s.code.number);
-break;
-}
-break;
-
-default:
- /* The c_generic backend or the c_mpi backend must be broken. */
-ncptl_fatal ("Internal error: unknown event type %d", thisev->type);
-break;
-}
-}
-}
-
-/* Finish up cleanly and return a status code. */
-static int conc_finalize (void)
-{
-int exitcode = 0;   /* Program exit code (to pass to exit()) */
-
- /* Declarations specific to the c_mpi backend */
-int mpiresult;   /* Return code from MPI_Finalize() */
-char log_key_str[128];   /* String representing the range of valid MPI tags */
-
- /* Finalization code specific to the c_mpi backend */
-log_key_str[0] = '\0';
-if (conc_mcast_tallies[CONC_MCAST_MPI_BCAST] > 0) {
-char onefuncstr[50];
-sprintf (onefuncstr, "%sMPI_Bcast()*%" NICS,
-log_key_str[0] == '\0' ? "" : " ", conc_mcast_tallies[CONC_MCAST_MPI_BCAST]);
-strcat (log_key_str, onefuncstr);
-}
-if (conc_mcast_tallies[CONC_MCAST_MPI_ALLTOALL] > 0) {
-char onefuncstr[50];
-sprintf (onefuncstr, "%sMPI_Alltoall()*%" NICS,
-log_key_str[0] == '\0' ? "" : " ", conc_mcast_tallies[CONC_MCAST_MPI_ALLTOALL]);
-strcat (log_key_str, onefuncstr);
-}
-if (conc_mcast_tallies[CONC_MCAST_MPI_ALLTOALLV] > 0) {
-char onefuncstr[50];
-sprintf (onefuncstr, "%sMPI_Alltoallv()*%" NICS,
-log_key_str[0] == '\0' ? "" : " ", conc_mcast_tallies[CONC_MCAST_MPI_ALLTOALLV]);
-strcat (log_key_str, onefuncstr);
-}
-if (log_key_str[0] != '\0')
-ncptl_log_add_comment ("Multicast functions used (statically)", log_key_str);
-
- /* Write a standard epilogue to the log file. */
-ncptl_log_commit_data (logstate);
-ncptl_log_write_epilogue (logstate);
-ncptl_log_close (logstate);
-
- /* Inform the run-time library that it's no longer needed. */
-ncptl_queue_empty (eventqueue);
-ncptl_free (eventqueue);
-ncptl_finalize();
-
- /* Finalization code specific to the c_mpi backend */
-mpiresult = MPI_Finalize();
-mpi_is_running = 0;
-exitcode = mpiresult!=MPI_SUCCESS;
-
- /* Return an exit status code. */
-return exitcode;
-}
-
-/*************************************************************************/
-/***************************** MAIN ROUTINE ******************************/
-/*************************************************************************/
-
-/* Program execution starts here. */
-int main (int argc, char *argv[])
-{
- /* Declare variables needed by all C-based backends. */
-CONC_EVENT * eventlist;   /* List of events to execute */
-ncptl_int numevents;   /* Number of entries in eventlist[] */
-
- /* ----- Initialization ----- */
-conc_initialize (argc, argv);
-eventlist = (CONC_EVENT *) ncptl_queue_contents (eventqueue, 0);
-numevents = ncptl_queue_length (eventqueue);
-sendrequests = (MPI_Request *) ncptl_queue_contents (sendreqQ, 0);
-sendstatuses = (MPI_Status *) ncptl_queue_contents (sendstatQ, 0);
-recvrequests = (MPI_Request *) ncptl_queue_contents (recvreqQ, 0);
-recvstatuses = (MPI_Status *) ncptl_queue_contents (recvstatQ, 0);
-starttime = ncptl_time();
-
- /* ----- Event-list processing ----- */
-conc_process_events (eventlist, 0, numevents-1, 1);
-
- /* ----- Finalization ----- */
-return conc_finalize();
-}

From 7c7b993d170eaf4064ee5c8693f868af3b19de49 Mon Sep 17 00:00:00 2001
From: Xin Wang <xwang149@hawk.iit.edu>
Date: Tue, 22 Jan 2019 15:12:10 -0600
Subject: [PATCH 06/15] Delete cosmoflow.ncptl

---
 scripts/conceptual_benchmarks/cosmoflow.ncptl | 28 -------------------
 1 file changed, 28 deletions(-)
 delete mode 100644 scripts/conceptual_benchmarks/cosmoflow.ncptl

diff --git a/scripts/conceptual_benchmarks/cosmoflow.ncptl b/scripts/conceptual_benchmarks/cosmoflow.ncptl
deleted file mode 100644
index b6895d79..00000000
--- a/scripts/conceptual_benchmarks/cosmoflow.ncptl
+++ /dev/null
@@ -1,28 +0,0 @@
-# Measure the performance of MPI_Allgather()
-# By Scott Pakin <pakin@lanl.gov>
-#
-# N.B. Requires the c_mpi backend.
-Require language version "1.5".
-# Parse the command line.
-numwords is "Message size (words)" and comes from "--msgsize" or "-s" with default 28825K.
-reps is "Number of repetitions" and comes from "--reps" or "-r" with default 100.
-computetime is "Computation time (ms)" and comes from "--compute" or "-c" with default 129.
-
-# Allocate a send buffer and a receive buffer.
-Task 0 multicasts a numwords*num_tasks word message from buffer 0 to all other tasks.
-Task 0 multicasts a numwords*num_tasks word message from buffer 1 to all other tasks.
-
-# Measure the performance of MPI_Allreduce().
-Task 0 resets its counters then
-for reps repetitions {
-    all tasks COMPUTES FOR computetime MILLISECONDS then
-    all tasks backend execute "
-        MPI_Allreduce([MESSAGE BUFFER 0], [MESSAGE BUFFER 1], (int)" and numwords and ",
-                MPI_INT, MPI_SUM, MPI_COMM_WORLD);
-        " then
-    all tasks backend execute "
-        MPI_Allreduce([MESSAGE BUFFER 0], [MESSAGE BUFFER 1], (int)" and numwords and ",
-                MPI_INT, MPI_SUM, MPI_COMM_WORLD);
-        "
-} then
-task 0 logs elapsed_usecs/1000 as "Elapse time (ms)".

From b6942329d1f831f915525438e587af89d999dd83 Mon Sep 17 00:00:00 2001
From: Xin Wang <xwang149@hawk.iit.edu>
Date: Tue, 22 Jan 2019 15:12:24 -0600
Subject: [PATCH 07/15] Delete hotpotato.c

---
 scripts/conceptual_benchmarks/hotpotato.c | 1258 ---------------------
 1 file changed, 1258 deletions(-)
 delete mode 100644 scripts/conceptual_benchmarks/hotpotato.c

diff --git a/scripts/conceptual_benchmarks/hotpotato.c b/scripts/conceptual_benchmarks/hotpotato.c
deleted file mode 100644
index a1409ffe..00000000
--- a/scripts/conceptual_benchmarks/hotpotato.c
+++ /dev/null
@@ -1,1258 +0,0 @@
-/**********************************************************************
- * This file was generated by coNCePTuaL on Mon Jan  7 23:10:01 2019
- * using the c_mpi backend (C + MPI).
- * Do not modify this file; modify /Users/xin/macworkspace/codes-dev/codes/scripts/conceptual_benchmarks/hotpotato.ncptl instead.
- *
- * Entire source program
- * ---------------------
- *   # Virtual ring "hot potato" test
- *   
- *   Require language version "1.5".
- *   
- *   trials is "Number of trials to perform" and comes from "--trials" or
- *     "-t" with default 100000.
- *   
- *   Assert that "the hot-potato test requires at least two tasks" with num_tasks>=2.
- *   
- *   Let len be 0 while {
- *     for each task_count in {2, ..., num_tasks} {
- *       task 0 outputs "Performing " and trials and " " and
- *   		   task_count and "-task runs...." then
- *       for trials repetitions plus 5 warmup repetitions and a synchronization {
- *         task 0 resets its counters then
- *         task 0 sends a len byte message to unsuspecting task 1 then
- *         task (n+1) mod task_count receives a len byte message from task n such that n<task_count then
- *         task n such that n>0 /\ n<task_count sends a len byte message to unsuspecting task (n+1) mod task_count then
- *         task 0 logs the task_count as "# of tasks" and
- *   		  the minimum of elapsed_usecs/task_count as "Latency (usecs)" and
- *   		  the mean of elapsed_usecs/task_count as "Latency (usecs)" and
- *   		  the variance of elapsed_usecs/task_count as "Latency (usecs)"
- *       } then
- *       task 0 computes aggregates
- *     }
- *   }
- **********************************************************************/
-
-/*****************
- * Include files *
- *****************/
-
-/* Header files needed by all C-based backends */
-#include <stdio.h>
-#include <string.h>
-#include <ncptl/ncptl.h>
-
-/* Header files specific to the c_mpi backend */
-#include <mpi.h>
-#include <stdarg.h>
-
-/**********
- * Macros *
- **********/
-
-/* Define the maximum loop trip count that we're willing to unroll fully. */
-#define CONC_MAX_UNROLL 5
-
-/* Specify the minimum number of trial iterations in each FOR <time> loop. */
-#define CONC_FOR_TIME_TRIALS 1
-
-/* Define a macro that rounds a double to a ncptl_int. */
-#define CONC_DBL2INT(D) ((ncptl_int)((D)+0.5))
-
-/* Define a macro that increments a buffer pointer by a byte offset. */
-#define CONC_GETBUFPTR(S) ((void *)((char *)thisev->s.S.buffer + thisev->s.S.bufferofs))
-
-/* Implement ncptl_func_task_of in terms of ncptl_physical_to_virtual. */
-#define ncptl_func_task_of(P) ((ncptl_int)(P) < 0LL || (ncptl_int)(P) >= var_num_tasks ? -1LL : ncptl_physical_to_virtual (procmap, (ncptl_int)(P)))
-#define ncptl_dfunc_task_of(P) ((double) ncptl_func_task_of(P))
-
-/* Implement ncptl_func_processor_of in terms of ncptl_virtual_to_physical. */
-#define ncptl_func_processor_of(V) ((ncptl_int)(V) < 0LL || (ncptl_int)(V) >= var_num_tasks ? -1LL : ncptl_virtual_to_physical (procmap, (ncptl_int)(V)))
-#define ncptl_dfunc_processor_of(V) ((double) ncptl_func_processor_of(V))
-
-/* Define a macro that increments REDUCE's alternate buffer pointer by a byte offset. */
-#define CONC_GETALTBUFPTR(S) ((void *)((char *)thisev->s.S.altbuffer + thisev->s.S.bufferofs))
-
-/* Estimate the number of unique communicators that this program will need.
- * (The tradeoff is one of initialization time versus memory consumption.) */
-#define ESTIMATED_COMMUNICATORS 128
-
-/* Specify an operation to use for all reduction operations. */
-#define REDUCE_OPERATION MPI_SUM
-#define REDUCE_OPERATION_NAME "MPI_SUM"
-
-
-/*********************
- * Type declarations *
- *********************/
-
-/* Enumerate the various mechanisms used to implement MULTICAST statements. */
-typedef enum {
-CONC_MCAST_MPI_BCAST,       /* One to many */
-CONC_MCAST_MPI_ALLTOALL,    /* Many to many, same data to all */ 
-CONC_MCAST_MPI_ALLTOALLV,   /* General many to many */
-CONC_MCAST_MPI_NUM_FUNCS    /* Number of the above */
-} CONC_MCAST_MPI_FUNC;
-
-/* Define the type of event to perform. */
-typedef enum {
-EV_SEND,     /* Synchronous send */
-EV_ASEND,    /* Asynchronous send */
-EV_RECV,     /* Synchronous receive */
-EV_ARECV,    /* Asynchronous receive */
-EV_WAIT,     /* Wait for all asynchronous sends/receives to complete */
-EV_DELAY,    /* Spin or sleep */
-EV_TOUCH,    /* Touch a region of memory */
-EV_SYNC,     /* Barrier synchronization */
-EV_RESET,    /* Reset counters */
-EV_STORE,    /* Store all counters' current values */
-EV_RESTORE,  /* Restore the previously pushed counter values */
-EV_FLUSH,    /* Compute aggregate functions for log-file columns */
-EV_MCAST,    /* Synchronous multicast */
-EV_REDUCE,   /* Reduction with or without a subsequent multicast */
-EV_BTIME,    /* Beginning of a timed loop */
-EV_ETIME,    /* Ending of a timed loop */
-EV_REPEAT,   /* Repeatedly process the next N events */
-EV_SUPPRESS, /* Suppress writing to the log and standard output */
-EV_NEWSTMT,  /* Beginning of a new top-level statement */
-EV_CODE,     /* None of the above */
-NUM_EVS      /* Number of event types in CONC_EVENT_TYPE */
-} CONC_EVENT_TYPE;
-
-/* Describe a synchronous or asynchronous send event. */
-typedef struct {
-ncptl_int dest;         /* Destination task */
-ncptl_int size;         /* Number of bytes to send */
-ncptl_int alignment;    /* Message alignment (in bytes) */
-ncptl_int pendingsends; /* # of outstanding sends */
-ncptl_int pendingrecvs; /* # of outstanding receives */
-ncptl_int buffernum;    /* Buffer # to send from */
-ncptl_int bufferofs;    /* Byte offset into the message buffer */
-ncptl_int tag;          /* Tag to use for selective receives */
-int misaligned;         /* 1=misaligned from a page; 0=align as specified */
-int touching;           /* 1=touch every word before sending */
-int verification;       /* 1=fill message buffer with known contents */
-void *buffer;           /* Pointer to message memory */
-MPI_Request * handle;   /* MPI handle representing an asynchronous send */
-} CONC_SEND_EVENT;
-
-/* Describe a synchronous or asynchronous receive event. */
-typedef struct {
-ncptl_int source;       /* Source task */
-ncptl_int size;         /* Number of bytes to receive */
-ncptl_int alignment;    /* Message alignment (in bytes) */
-ncptl_int pendingsends; /* # of outstanding sends */
-ncptl_int pendingrecvs; /* # of outstanding receives */
-ncptl_int buffernum;    /* Buffer # to receive into */
-ncptl_int bufferofs;    /* Byte offset into the message buffer */
-ncptl_int tag;          /* Tag to use for selective receives */
-int misaligned;         /* 1=misaligned from a page; 0=align as specified */
-int touching;           /* 1=touch every word after reception */
-int verification;       /* 1=verify that all bits are correct */
-void *buffer;           /* Pointer to message memory */
-MPI_Request * handle;   /* MPI handle representing an asynchronous receive */
-} CONC_RECV_EVENT;
-
-/* Describe a wait-for-asynchronous-completions event. */
-typedef struct {
-ncptl_int numsends;     /* # of sends we expect to complete. */
-ncptl_int numrecvs;     /* # of receives we expect to complete. */
-ncptl_int numrecvbytes; /* # of bytes we expect to receive-complete */
-ncptl_int *touchedlist;    /* List of receives that need to be touched */
-ncptl_int numtouches;         /* # of elements in the above */
-} CONC_WAIT_EVENT;
-
-/* Describe a spin or sleep delay. */
-typedef struct {
-uint64_t microseconds;  /* Length of delay in microseconds */
-int spin0sleep1;        /* 0=spin; 1=sleep */
-} CONC_DELAY_EVENT;
-
-/* Describe a barrier synchronization event. */
-typedef struct {
-MPI_Comm communicator;   /* Set of tasks to synchronize */
-} CONC_SYNC_EVENT;
-
-/* Describe a walk over a memory-region. */
-typedef struct {
-ncptl_int regionbytes;  /* Size in bytes of the region to touch */
-ncptl_int bytestride;   /* Stride in bytes to touch */
-ncptl_int numaccesses;  /* Number of words to touch */
-ncptl_int wordsize;     /* Size in bytes of each touch */
-ncptl_int firstbyte;    /* Byte offset of the first byte to touch */
-} CONC_TOUCH_EVENT;
-
-/* Describe a synchronous multicast event. */
-typedef struct {
-ncptl_int source;       /* Source task, -1 in the many-to-many case */
-ncptl_int size;         /* Number of bytes to send */
-ncptl_int alignment;    /* Message alignment (in bytes) */
-ncptl_int pendingsends; /* # of outstanding sends */
-ncptl_int pendingrecvs; /* # of outstanding receives */
-ncptl_int buffernum;    /* Buffer # to send/receive from */
-ncptl_int bufferofs;    /* Byte offset into the message buffer */
-ncptl_int tag;          /* Tag to use for selective receives */
-int misaligned;         /* 1=misaligned from a page; 0=align as specified */
-int touching;           /* 1=touch every word before sending */
-int verification;       /* 1=verify that all bits are correct */
-void *buffer;           /* Pointer to message memory */
-ncptl_int size2;   /* Number of bytes to receive in the many-to-many case */
-ncptl_int bufferofs2;   /* Byte offset into the message buffer in the many-to-many case */
-void * buffer2;   /* Pointer to receive-message memory in the many-to-many case */
-MPI_Comm communicator;   /* Set of tasks to multicast to/from */
-int root;   /* source's rank within communicator */
-int * sndvol;   /* Volume of data to send to each rank in the communicator */
-int * snddisp;   /* Offset from buffer of each message to send */
-int * rcvvol;   /* Volume of data to receive from each rank in the communicator */
-int * rcvdisp;   /* Offset from buffer2 of each message to receive */
-CONC_MCAST_MPI_FUNC mpi_func;   /* MPI function to use to perform the multicast */
-} CONC_MCAST_EVENT;
-
-/* Describe a reduction event. */
-typedef struct {
-ncptl_int numitems;     /* # of items to reduce */
-ncptl_int itemsize;     /* # of bytes per item */
-ncptl_int alignment;    /* Message alignment (in bytes) */
-ncptl_int pendingsends; /* # of outstanding sends */
-ncptl_int pendingrecvs; /* # of outstanding receives */
-ncptl_int buffernum;    /* Buffer # to send/receive from */
-ncptl_int bufferofs;    /* Byte offset into the message buffer */
-ncptl_int tag;          /* Tag to use for selective receives */
-int misaligned;         /* 1=misaligned from a page; 0=align as specified */
-int touching;           /* 1=touch every word before sending/after receiving */
-int sending;            /* 1=we're a sender */
-int receiving;          /* 1=we're a receiver */
-void *buffer;           /* Pointer to message memory */
-void * altbuffer;   /* Pointer to additional message memory */
-MPI_Comm sendcomm;   /* Set of tasks to reduce from */
-MPI_Comm recvcomm;   /* Set of tasks to reduce to */
-MPI_Datatype datatype;   /* MPI datatype to reduce */
-int reducetype;   /* 0=reduce; 1=allreduce; 2=reduce+bcast */
-ncptl_int reduceroot;   /* Root task of the reduction if reducetype is 0 or 2 */
-ncptl_int bcastroot;   /* Root task of the multicast if reducetype is 2 */
-} CONC_REDUCE_EVENT;
-
-/* Describe an event representing the beginning of a timed loop. */
-typedef struct {
-uint64_t usecs;         /* Requested loop duration */
-uint64_t warmup_usecs;  /* Requested duration of warmup loops */
-uint64_t starttime;     /* Time at which the loop state last changed */
-uint64_t itersleft;     /* # of iterations remaining */
-uint64_t previters;     /* # of iterations we performed last time */
-int prev_quiet;         /* Previous value of suppress_output */
-int timing_trial;       /* 1=performing a timing trial; 0=running for real */
-volatile int finished;  /* 1=time has expired; 0=still ticking */
-} CONC_BTIME_EVENT;
-
-/* Describe an event representing the end of a timed loop. */
-typedef struct {
-ncptl_int begin_event;  /* Index into eventlist[] of the corresponding BTIME event */
-} CONC_ETIME_EVENT;
-
-/* Describe an event representing repetitions of subsequent events. */
-typedef struct {
-ncptl_int end_event;    /* Index into eventlist[] of the last event to repeat */
-ncptl_int numreps;      /* # of repetitions to perform */
-} CONC_REPEAT_EVENT;
-
-/* Describe an event representing output suppression (either on or off). */
-typedef struct conc_suppress_event {
-int quiet;              /* 0=allow output; 1=suppress it */
-int prev_quiet;         /* Previous value of suppress_output */
-ncptl_int matching_event;  /* Event ID of the "suppression on" event */
-uint64_t stop_elapsed_usecs;   /* Time at which we suppressed output */
-} CONC_SUPPRESS_EVENT;
-
-/* Describe an event representing arbitrary code to execute at run time. */
-typedef struct {
-ncptl_int number;       /* Unique number corresponding to a specific piece of code */NCPTL_VIRT_PHYS_MAP *procmap;  /* Current mapping between tasks and processors */
-ncptl_int var_task_count;   /* Copy of var_task_count to use within a piece of code */
-ncptl_int var_trials;   /* Copy of var_trials to use within a piece of code */
-} CONC_CODE_EVENT;
-
-/* Describe an arbitrary coNCePTuaL event. */
-typedef struct {
-CONC_EVENT_TYPE type;          /* Type of event */
-union {
-CONC_SEND_EVENT send;          /* Send state */
-CONC_RECV_EVENT recv;          /* Receive state */
-CONC_WAIT_EVENT wait;          /* Wait-for-completions state */
-CONC_DELAY_EVENT delay;        /* State for spins and sleeps */
-CONC_TOUCH_EVENT touch;        /* State for memory touching */
-CONC_SYNC_EVENT sync;          /* Synchronization state */
-CONC_MCAST_EVENT mcast;        /* Multicast state */
-CONC_REDUCE_EVENT reduce;      /* Reduction state */
-CONC_BTIME_EVENT btime;        /* Timed-loop state */
-CONC_ETIME_EVENT etime;        /* Additional timed-loop state */
-CONC_REPEAT_EVENT rep;         /* Repeated-events state */
-CONC_SUPPRESS_EVENT suppress;  /* State for suppressing output */
-CONC_CODE_EVENT code;          /* State for arbitrary code */
-} s;
-} CONC_EVENT;
-
-/* Fully specify an arbitrary for() loop (used by FOR EACH). */
-typedef struct {
-NCPTL_QUEUE *list_comp;  /* NULL=ordinary list; other=list comprehension values */
-int integral;        /* 1=integral values; 0=floating-point values */
-enum {               /* Comparison of loop variable to end variable */
-CONC_LEQ,                /* Increasing progression */
-CONC_GEQ                 /* Decreasing progression */
-} comparator;
-enum {               /* How to increment the loop variable */
-CONC_ADD,                /* Arithmetically */
-CONC_MULT,               /* Geometrically increasing */
-CONC_DIV                 /* Geometrically decreasing */
-} increment;
-union {
-struct {
-ncptl_int loopvar;   /* Loop variable */
-ncptl_int prev_loopvar; /* Previous value of loop variable */
-ncptl_int startval;  /* Initial value of loop variable */
-ncptl_int endval;    /* Value not to exceed */
-ncptl_int incval;    /* Loop-variable increment */
-} i;
-struct {
-double loopvar;      /* Loop variable */
-double prev_loopvar; /* Previous value of loop variable */
-double startval;     /* Initial value of loop variable */
-double endval;       /* Value not to exceed */
-double incval;       /* Loop-variable increment */
-} d;
-} u;
-} LOOPBOUNDS;
-
-/********************
- * Global variables *
- ********************/
-
-/* Variables exported to coNCePTuaL programs */
-static ncptl_int var_bytes_received = 0;   /* Total number of bytes received */
-static ncptl_int var_msgs_received = 0;   /* Total number of messages received */
-static ncptl_int var_bit_errors = 0;   /* Total number of bit errors observed */
-static ncptl_int var_total_msgs = 0;   /* Sum of messages sent and messages received */
-static ncptl_int var_msgs_sent = 0;   /* Total number of messages sent */
-static ncptl_int var_bytes_sent = 0;   /* Total number of bytes sent */
-static ncptl_int var_num_tasks = 1;   /* Number of tasks running the program */
-static ncptl_int var_elapsed_usecs = 0;   /* Elapsed time in microseconds */
-static ncptl_int var_total_bytes = 0;   /* Sum of bytes sent and bytes received */
-
-/* Dummy variable to help mark other variables as used */
-union {
-ncptl_int ni;
-int i;
-void *vp;
-} conc_dummy_var;
-
-/* Variables used internally by boilerplate code */
-static uint64_t starttime;   /* Time the clock was last reset (microseconds) */
-static ncptl_int pendingrecvs = 0;   /* Current # of outstanding receives */
-static ncptl_int pendingrecvbytes = 0; /* Current # of bytes in outstanding receives */
-static NCPTL_QUEUE *touchedqueue;      /* Queue of asynchronous receives to touch */
-static ncptl_int pendingsends = 0;   /* Current # of outstanding sends */
-static NCPTL_QUEUE *eventqueue;   /* List of coNCePTuaL events to perform */
-static int within_time_loop = 0;   /* 1=we're within a FOR <time> loop */
-static int suppress_output = 0;    /* 1=suppress output to stdout and the log file */
-static void *touch_region = NULL;   /* Memory region to touch */
-static ncptl_int touch_region_size = 0;   /* # of bytes in the above */
-static int virtrank;    /* This task's virtual rank in the computation */
-static int physrank;    /* This task's physical rank in the computation */
-static NCPTL_VIRT_PHYS_MAP *procmap;  /* Virtual to physical rank mapping */
-static NCPTL_LOG_FILE_STATE *logstate;   /* Opaque object representing all log-file state */
-static char *logfile_uuid;   /* Execution UUID to write to every log file */
-static char *logfiletmpl;   /* Template for the log file's name */
-static char *logfiletmpl_default;   /* Default value of the above */
-
-/* Global variables specific to the c_mpi backend */
-static ncptl_int mpi_is_running = 0;   /* 1=MPI has been initialized */
-static NCPTL_QUEUE * recvreqQ;   /* List of MPI receive requests */
-static MPI_Request * recvrequests;   /* List version of recvreqQ */
-static NCPTL_QUEUE * recvstatQ;   /* List of MPI receive statuses */
-static MPI_Status * recvstatuses;   /* List version of recvstatQ */
-static NCPTL_QUEUE * sendreqQ;   /* List of MPI send requests */
-static MPI_Request * sendrequests;   /* List version of sendreqQ */
-static NCPTL_QUEUE * sendstatQ;   /* List of MPI send statuses */
-static MPI_Status * sendstatuses;   /* List version of sendstatQ */
-static NCPTL_SET * communicators;   /* Map from an array of processor flags to an MPI communicator */
-static MPI_Errhandler mpi_error_handler;   /* Handle to handle_MPI_error() */
-static ncptl_int mpi_tag_ub;   /* Upper bound on an MPI tag value */
-static ncptl_int conc_mcast_tallies[CONC_MCAST_MPI_NUM_FUNCS] = {0};   /* Tallies of (static) multicast implementation functions */
-
-/* Program-specific variables */
-ncptl_int var_trials;   /* Number of trials to perform (command-line argument) */
-
-/*************************
- * Function declarations *
- *************************/
-
-/* Make MPI errors invoke ncptl_fatal(). */
-static void handle_MPI_error (MPI_Comm *comm, int *errcode, ...)
-{
-va_list args;
-char errstring[MPI_MAX_ERROR_STRING];
-int errstrlen;
-
-va_start (args, errcode);
-if (MPI_Error_string (*errcode, errstring, &errstrlen) == MPI_SUCCESS)
-ncptl_fatal ("MPI run-time error: %s", errstring);
-else
-ncptl_fatal ("MPI aborted with unrecognized error code %d", *errcode);
-conc_dummy_var.vp = (void *) comm;   /* Prevent the compiler from complaining that comm is unused. */
-va_end (args);
-}
-
-/* Perform the equivalent of MPI_Comm_rank() for an arbitrary process. */
-static int rank_in_MPI_communicator (MPI_Comm subcomm, int global_rank)
-{
-  MPI_Group world_group;   /* Group associated with MPI_COMM_WORLD */
-  MPI_Group subgroup;      /* Group associate with subcomm */
-  int subrank;             /* global_rank's rank within subcomm */
-
-  MPI_Comm_group (MPI_COMM_WORLD, &world_group);
-  MPI_Comm_group (subcomm, &subgroup);
-  MPI_Group_translate_ranks (world_group, 1, &global_rank, subgroup, &subrank);
-  return subrank;
-}
-
-/* Map an arbitrary tag to within MPI's valid range of [0, mpi_tag_ub]. */
-static ncptl_int map_tag_into_MPI_range (ncptl_int tag)
-{
-if (tag == NCPTL_INT_MIN)
- /* Avoid taking the absolute value of NCPTL_INT_MIN. */
-tag = 555666773LL;   /* Arbitrary value */
-tag = ncptl_func_abs (tag);   /* Only nonnegatives values are allowed. */
-if (mpi_tag_ub < NCPTL_INT_MAX)
-tag %= mpi_tag_ub + 1;
-return tag;
-}
-
-/* Given an array of task in/out booleans return an MPI
- * communicator that represents the "in" tasks. */
-static MPI_Comm define_MPI_communicator (char *procflags)
-{
-MPI_Comm *existing_comm;    /* Previously defined MPI communicator */
-MPI_Comm new_comm;          /* Newly defined MPI communicator */
-
-existing_comm = (MPI_Comm *) ncptl_set_find (communicators, (void *)procflags);
-if (existing_comm)
-return *existing_comm;
-(void) MPI_Comm_split (MPI_COMM_WORLD, (int)procflags[physrank], physrank, &new_comm);
-(void) MPI_Errhandler_set (new_comm, mpi_error_handler);
-ncptl_set_insert (communicators, (void *)procflags, (void *)&new_comm);
-return define_MPI_communicator (procflags);
-}
-
-/* Return 1 if a sequence loop will take at least one trip. */
-static int conc_seq_nonempty (LOOPBOUNDS *seq)
-{
-ncptl_int startval;   /* Integer version of seq's startval element */
-ncptl_int endval;   /* Integer version of seq's endval element */
-
-if (seq->integral) {
-startval = seq->u.i.startval;
-endval = seq->u.i.endval;
-}
-else {
-startval = CONC_DBL2INT (seq->u.d.startval);
-endval = CONC_DBL2INT (seq->u.d.endval);
-}
-switch (seq->comparator) {
-case CONC_LEQ:
-return startval <= endval;
-
-case CONC_GEQ:
-return startval >= endval;
-
-default:
-ncptl_fatal ("Internal error -- unknown comparator");
-}
-return -1;     /* Appease idiotic compilers. */
-}
-
-/* Initialize a sequence loop. */
-static void conc_seq_init (LOOPBOUNDS *seq)
-{
-if (seq->integral) {
-seq->u.i.loopvar = seq->u.i.startval;
-seq->u.i.prev_loopvar = seq->u.i.loopvar - 1;
-}
-else {
-seq->u.d.loopvar = seq->u.d.startval;
-seq->u.d.prev_loopvar = seq->u.d.loopvar - 1.0;
-}
-}
-
-/* Return 1 if a sequence loop should continue, 0 when finished. */
-static int conc_seq_continue (LOOPBOUNDS *seq)
-{
-LOOPBOUNDS seq_int;   /* Integer equivalent of *seq */
-if (seq->integral)
-seq_int = *seq;
-else {
-seq_int.u.i.loopvar = CONC_DBL2INT (seq->u.d.loopvar);
-seq_int.u.i.prev_loopvar = CONC_DBL2INT (seq->u.d.prev_loopvar);
-seq_int.u.i.endval = CONC_DBL2INT (seq->u.d.endval);
-}
-
-if (seq_int.u.i.loopvar == seq_int.u.i.prev_loopvar)
-return 0;
-switch (seq->comparator) {
-case CONC_LEQ:
-return seq_int.u.i.loopvar <= seq_int.u.i.endval;
-
-case CONC_GEQ:
-return seq_int.u.i.loopvar >= seq_int.u.i.endval;
-
-default:
-ncptl_fatal ("Internal error -- unknown comparator");
-}
-return -1;     /* Appease idiotic compilers. */
-}
-
-/* Proceed to the next iteration of a sequence loop. */
-static void conc_seq_next (LOOPBOUNDS *seq)
-{
-if (seq->integral) {
-seq->u.i.prev_loopvar = seq->u.i.loopvar;
-switch (seq->increment) {
-case CONC_ADD:
-seq->u.i.loopvar += seq->u.i.incval;
-break;
-
-case CONC_MULT:
-seq->u.i.loopvar *= seq->u.i.incval;
-break;
-
-case CONC_DIV:
-seq->u.i.loopvar /= seq->u.i.incval;
-break;
-
-default:
-ncptl_fatal ("Internal error -- unknown incrementer");
-}
-}
-else {
-seq->u.d.prev_loopvar = seq->u.d.loopvar;
-switch (seq->increment) {
-case CONC_ADD:
-seq->u.d.loopvar += seq->u.d.incval;
-break;
-
-case CONC_MULT:
-seq->u.d.loopvar *= seq->u.d.incval;
-break;
-
-case CONC_DIV:
-seq->u.d.loopvar /= seq->u.d.incval;
-break;
-
-default:
-ncptl_fatal ("Internal error -- unknown incrementer");
-}
-}
-}
-
-/* Inhibit the compiler from complaining that
- * certain variables are defined but not used.
- * This function should never be called. */
-void conc_mark_variables_used (void)
-{
-conc_dummy_var.ni = var_bytes_received;
-conc_dummy_var.ni = var_msgs_received;
-conc_dummy_var.ni = var_bit_errors;
-conc_dummy_var.ni = var_total_msgs;
-conc_dummy_var.ni = var_msgs_sent;
-conc_dummy_var.ni = var_bytes_sent;
-conc_dummy_var.ni = var_num_tasks;
-conc_dummy_var.ni = var_elapsed_usecs;
-conc_dummy_var.ni = var_total_bytes;
-conc_dummy_var.ni = pendingrecvbytes;
-conc_dummy_var.ni = touch_region_size;
-conc_dummy_var.vp = touch_region;
-conc_dummy_var.i  = within_time_loop;
-conc_dummy_var.i  = suppress_output;
-rank_in_MPI_communicator (MPI_COMM_WORLD, 0);
-}
-
-/* Allocate a new event of a given type and return a pointer to it. */
-static CONC_EVENT *conc_allocate_event (CONC_EVENT_TYPE type)
-{
-CONC_EVENT *newevent = (CONC_EVENT *) ncptl_queue_allocate (eventqueue);
-
-newevent->type = type;
-return newevent;
-}
-
-/* Declare an exit handler that gets called automatically when the
- * program terminates, whether successfully or not. */
-static void conc_exit_handler (void)
-{
-if (mpi_is_running)
-MPI_Abort (MPI_COMM_WORLD, 1);
-}
-
-
-/* Initialize coNCePTuaL, the messaging layer, and this program itself. */
-static void conc_initialize (int argc, char *argv[])
-{
- /* Variables needed by all C-based backends */
-CONC_EVENT * eventlist;   /* List of events to execute */
-ncptl_int numevents;   /* Number of entries in eventlist[] */
-int help_only = 0;   /* 1=User specified --help; save time by skipping ncptl_init() */
-char * argv0 = strrchr(argv[0], '/') ? strrchr(argv[0], '/')+1 : argv[0];   /* Base name of the executable program */
-int i;   /* Generic loop variable */
-
- /* Declare all of our command-line arguments. */
-NCPTL_CMDLINE arguments[] = {
-{ NCPTL_TYPE_STRING, NULL, "logfile", 'L', "Log-file template", {0}},
-{ NCPTL_TYPE_INT, NULL, "trials", 't', "Number of trials to perform", {0}}
-};
-
- /* Incorporate the complete coNCePTuaL source code as an array
-  * for use by ncptl_log_write_prologue(). */
-char *sourcecode[] = {
-"# Virtual ring \"hot potato\" test",
-"",
-"Require language version \"1.5\".",
-"",
-"trials is \"Number of trials to perform\" and comes from \"--trials\" or",
-"  \"-t\" with default 100000.",
-"",
-"Assert that \"the hot-potato test requires at least two tasks\" with num_tasks>=2.",
-"",
-"Let len be 0 while {",
-"  for each task_count in {2, ..., num_tasks} {",
-"    task 0 outputs \"Performing \" and trials and \" \" and",
-"		   task_count and \"-task runs....\" then",
-"    for trials repetitions plus 5 warmup repetitions and a synchronization {",
-"      task 0 resets its counters then",
-"      task 0 sends a len byte message to unsuspecting task 1 then",
-"      task (n+1) mod task_count receives a len byte message from task n such that n<task_count then",
-"      task n such that n>0 /\\ n<task_count sends a len byte message to unsuspecting task (n+1) mod task_count then",
-"      task 0 logs the task_count as \"# of tasks\" and",
-"		  the minimum of elapsed_usecs/task_count as \"Latency (usecs)\" and",
-"		  the mean of elapsed_usecs/task_count as \"Latency (usecs)\" and",
-"		  the variance of elapsed_usecs/task_count as \"Latency (usecs)\"",
-"    } then",
-"    task 0 computes aggregates",
-"  }",
-"}",
-NULL
-};
-
- /* Variables specific to the c_mpi backend */
-int num_tasks;   /* int version of var_num_tasks needed by MPI_Comm_size() */
-char * procflags;   /* Array of 1s representing an all-task MPI communicator */
-MPI_Comm comm_world = MPI_COMM_WORLD;   /* Copy of MPI_COMM_WORLD that we can take the address of */
-void * attr_val;   /* Pointed to the value of MPI_TAG_UB */
-int attr_flag = 0;   /* true=MPI_TAG_UB was extracted; false=not extracted */
-char log_key_str[128];   /* String representing the range of valid MPI tags */
-
- /* As a special case, if the command line contains --help, then skip
-  * the coNCePTuaL initialization step. */
-for (i=1; i<argc; i++)
-if (!strcmp(argv[i], "--"))
-break;
-else
-if (!strcmp(argv[i], "--help") || !strcmp(argv[i], "-?")) {
-argv[1] = "-?";   /* Guaranteed to work, even with getopt() */
-help_only = 1;
-break;
-}
-
- /* Perform various initializations specific to the c_mpi backend. */
- /* Initialize MPI. */
-(void) MPI_Init(&argc, &argv);
-mpi_is_running = 1;
-
- /* Initialize the coNCePTuaL run-time library. */
-if (!help_only)
-ncptl_init (NCPTL_RUN_TIME_VERSION, argv[0]);
-(void) atexit (conc_exit_handler);
-
- /* Initialize the communication routines needed by the c_mpi backend. */
-(void) MPI_Errhandler_create ((MPI_Handler_function *)handle_MPI_error, &mpi_error_handler);
-(void) MPI_Errhandler_set (MPI_COMM_WORLD, mpi_error_handler);
-(void) MPI_Comm_rank(MPI_COMM_WORLD, &physrank);
-(void) MPI_Comm_size(MPI_COMM_WORLD, &num_tasks);
-var_num_tasks = (ncptl_int) num_tasks;
-(void) MPI_Comm_get_attr(MPI_COMM_WORLD, MPI_TAG_UB, &attr_val, &attr_flag);
-mpi_tag_ub = (ncptl_int) (attr_flag ? *(int *)attr_val : 32767);
-
- /* Generate and broadcast a UUID. */
-logfile_uuid = ncptl_log_generate_uuid();
-(void) MPI_Bcast ((void *)logfile_uuid, 37, MPI_CHAR, 0, MPI_COMM_WORLD);
-
- /* Plug variables and default values into the NCPTL_CMDLINE structure. */
-arguments[0].variable = (CMDLINE_VALUE *) &logfiletmpl;
-arguments[1].variable = (CMDLINE_VALUE *) &var_trials;
-arguments[1].defaultvalue.intval = 100000LL;
-logfiletmpl_default = (char *) ncptl_malloc (strlen(argv0) + 15, 0);
-sprintf (logfiletmpl_default, "%s-%%p.log", argv0);
-arguments[0].defaultvalue.stringval = logfiletmpl_default;
-
- /* Parse the command line. */
-mpi_is_running = 0;   /* Don't invoke MPI_Abort() after --help. */
-ncptl_parse_command_line (argc, argv, arguments, sizeof(arguments)/sizeof(NCPTL_CMDLINE));
-if (help_only)
-ncptl_fatal ("Internal error in the c_generic backend: failed to exit after giving help");
-mpi_is_running = 1;
-
- /* Establish a mapping from (virtual) task IDs to (physical) ranks. */
-procmap = ncptl_allocate_task_map (var_num_tasks);
-virtrank = ncptl_physical_to_virtual (procmap, physrank);
-
- /* Perform initializations specific to the c_mpi backend. */
-ncptl_log_add_comment ("MPI send routines", "MPI_Send() and MPI_Isend()");
-ncptl_log_add_comment ("MPI reduction operation", REDUCE_OPERATION_NAME);
-sprintf (log_key_str, "[0, %" NICS "]", mpi_tag_ub);
-ncptl_log_add_comment ("MPI tag range", log_key_str);
-
- /* Open the log file and write some standard prologue information to it. */
-logstate = ncptl_log_open (logfiletmpl, physrank);
-ncptl_log_write_prologue (logstate, argv[0], logfile_uuid, "c_mpi", "C + MPI",
-var_num_tasks,
-arguments, sizeof(arguments)/sizeof(NCPTL_CMDLINE),
-sourcecode);
-ncptl_free (logfile_uuid);
-
- /* Allocate a variety of dynamically growing queues. */
-eventqueue = ncptl_queue_init (sizeof (CONC_EVENT));
-touchedqueue = ncptl_queue_init (sizeof (ncptl_int));
-
- /* Perform initializations specific to the c_mpi backend. */
-sendreqQ = ncptl_queue_init (sizeof (MPI_Request));
-sendstatQ = ncptl_queue_init (sizeof (MPI_Status));
-recvreqQ = ncptl_queue_init (sizeof (MPI_Request));
-recvstatQ = ncptl_queue_init (sizeof (MPI_Status));
-communicators = ncptl_set_init (ESTIMATED_COMMUNICATORS, var_num_tasks*sizeof(char), sizeof(MPI_Comm));procflags = (char *) ncptl_malloc (var_num_tasks*sizeof(char), 0);
-for (i=0; i<var_num_tasks; i++)
-procflags[i] = 1;
-ncptl_set_insert (communicators, (void *)procflags, (void *)&comm_world);
-ncptl_free (procflags);
-
- /****************************************************
-  * Generated, program-specific initialization code. *
-  ****************************************************/
-
- /* ASSERT THAT the hot-potato test requires at least two tasks. */
-if (!((var_num_tasks)>=(2LL)))
-ncptl_fatal ("Assertion failure: the hot-potato test requires at least two tasks");
-{
- /* LET var_len BE 0LL WHILE... */
-ncptl_int var_len = 0LL;
-
- /* FOR EACH var_task_count IN [(['2LL'], 'var_num_tasks')]... */
-{
-LOOPBOUNDS loopbounds[1];   /* List of range descriptions */
-ncptl_int rangenum;   /* Current offset into loopbounds[] */
-ncptl_int initial_vals[1];   /* Cache of the initial, enumerated values */
-ncptl_int final_val;   /* Cache of the final value */
-
- /* Write range 0's loop bounds, "next" function, and termination function to loopbounds[0]. */
-loopbounds[0].list_comp = NULL;
-initial_vals[0] = 2LL;
-final_val = var_num_tasks;
-loopbounds[0].integral = 1;
-loopbounds[0].u.i.startval = initial_vals[0];
-loopbounds[0].u.i.endval = final_val;
-loopbounds[0].comparator = initial_vals[0]<=final_val ? CONC_LEQ : CONC_GEQ;
-loopbounds[0].increment = CONC_ADD;
-loopbounds[0].u.i.incval = initial_vals[0]<=final_val ? 1LL : -1LL;
-
- /* Now that we've defined all of our ranges we iterate over each range
-  * and each element within each range. */
-for (rangenum=0LL; rangenum<1; rangenum++) {
-LOOPBOUNDS * thisrange = &loopbounds[rangenum];   /* Current range */
-if (conc_seq_nonempty (thisrange))
-for (conc_seq_init (thisrange);
-conc_seq_continue (thisrange);
-conc_seq_next (thisrange)) {
-ncptl_int var_task_count;
-if (thisrange->list_comp == NULL)
- /* The loopvar field contains the value for var_task_count. */
-var_task_count = thisrange->integral ? thisrange->u.i.loopvar : CONC_DBL2INT(thisrange->u.d.loopvar);
-else
- /* The queue of list-comprehension values provides the value for var_task_count. */
-var_task_count = *(ncptl_int *)ncptl_queue_pop(thisrange->list_comp);
-{
- /* TASK 0LL OUTPUTS ['"Performing "', 'var_trials', '" "', 'var_task_count', '"-task runs...."'] */
-if ((0LL) == virtrank) {   /* TASK 0LL */
- /* The current coNCePTuaL statement applies to our task. */
-CONC_EVENT *thisev = conc_allocate_event (EV_CODE);
-thisev->s.code.number = 0;
-thisev->s.code.procmap = NULL;
-thisev->s.code.var_task_count = var_task_count;
-thisev->s.code.var_trials = var_trials;
-}
- /* THEN... */
- /* FOR var_trials REPETITIONS... */
-{
-ncptl_int loopnum;   /* 0=warmup repetitions; 1=main repetitions */
-
-for (loopnum=0LL; loopnum<2LL; loopnum++) {
-ncptl_int numreps = loopnum==0LL ? (5LL) : (var_trials);   /* Total # of repetitions to perform */
-int unroll_loop = numreps <= CONC_MAX_UNROLL;   /* 1=unroll loop; 0=use a REPEAT event */
-ncptl_int repnum;   /* Current repetition number */
-CONC_EVENT * repevent = NULL;   /* Event designating repetition */
-ncptl_int repeventnum = -1LL;   /* Event number corresponding to repevent */
-ncptl_int on_event = -1;   /* Event ID of the "suppression on" event */
-
- /* Suppress output at the start of the warmup repetitions. */
-if (numreps > 0LL && loopnum == 0LL) {
-CONC_EVENT * thisev_on;   /* Output-suppression event */
-thisev_on =  conc_allocate_event (EV_SUPPRESS);
-thisev_on->s.suppress.quiet = 1;
-on_event = ncptl_queue_length(eventqueue) - 1;
-}
-
- /* Conditionally unroll the loop. */
-for (repnum=0; repnum<(unroll_loop?numreps:1); repnum++) {
- /* Allocate a repeat event if we have more than one repetition. */
-if (!unroll_loop && numreps > 1LL) {
-repeventnum = ncptl_queue_length (eventqueue);
-repevent =  conc_allocate_event (EV_REPEAT);
-repevent->s.rep.numreps = numreps;
-}
-
- /* Output a loop body if we have at least one repetition. */
-if (unroll_loop || numreps > 0LL) {
- /* TASK 0LL RESET THEIR COUNTERS */
-if ((0LL) == virtrank) {   /* TASK 0LL */
- /* The current coNCePTuaL statement applies to our task. */
-(void) conc_allocate_event (EV_RESET);
-}
- /* THEN... */
- /* TASK 0LL SENDS TO TASK 1LL */
-if ((0LL) == virtrank) {   /* TASK 0LL */
- /* The current coNCePTuaL statement applies to our task. */
-ncptl_int virtdest = 1LL;
-if (virtdest>=0 && virtdest<var_num_tasks) {
- /* In this scope, 1LL represents a single receiver. */
- /* Prepare to send 1LL messages. */
-ncptl_int numreps = 1LL;   /* Number of messages */
-if (numreps > 1LL) {
-CONC_EVENT * repeatev;   /* Event specifying the number of repetitions to perform */
-repeatev = conc_allocate_event (EV_REPEAT);
-repeatev->s.rep.end_event = ncptl_queue_length(eventqueue);
-repeatev->s.rep.numreps = numreps;
-}
-
- /* Ensure we have at least one message to send. */
-if (numreps > 0LL) {
-CONC_EVENT *thisev = conc_allocate_event (EV_SEND);
-
- /* Fill in all of the fields of a send-event structure. */
-thisev->s.send.dest = ncptl_virtual_to_physical (procmap, 1LL);
-thisev->s.send.size = var_len;
-thisev->s.send.alignment = 0LL;
-thisev->s.send.misaligned = 0;
-thisev->s.send.touching = 0;
-thisev->s.send.verification = 0;
-thisev->s.send.tag = 0LL;
-thisev->s.send.pendingsends = pendingsends;
-thisev->s.send.pendingrecvs = pendingrecvs;
-thisev->s.send.bufferofs = 0LL;
-thisev->s.send.buffernum = thisev->s.send.verification ? pendingsends+pendingrecvs : 0;
-(void) ncptl_malloc_message (thisev->s.send.size+thisev->s.send.bufferofs, thisev->s.send.alignment, thisev->s.send.buffernum, thisev->s.send.misaligned);
-thisev->s.send.buffer = NULL;
-thisev->s.send.tag = map_tag_into_MPI_range (thisev->s.send.tag);
-}
-}
-}
- /* THEN... */
- /* TASKS var_n SUCH THAT (var_n)<(var_task_count) RECEIVES FROM TASK ncptl_func_modulo((var_n)+(1LL),var_task_count) */
-{
-ncptl_int var_n;
- /* Loop over all tasks to see which will send to us. */
-for (var_n=0; var_n<var_num_tasks; var_n++)
-if ((var_n)<(var_task_count)) {
- /* var_n now represents one of the tasks that will send to us. */
-ncptl_int ivar_a_task = var_n;
-if (virtrank == (ncptl_func_modulo((var_n)+(1LL),var_task_count))) {
- /* In this scope, we must be a message recipient. */
- /* Prepare to receive 1LL messages. */
-ncptl_int numreps = 1LL;   /* Number of messages */
-if (numreps > 1LL) {
-CONC_EVENT * repeatev;   /* Event specifying the number of repetitions to perform */
-repeatev = conc_allocate_event (EV_REPEAT);
-repeatev->s.rep.end_event = ncptl_queue_length(eventqueue);
-repeatev->s.rep.numreps = numreps;
-}
-
- /* Ensure we have at least one message to receive. */
-if (numreps > 0LL) {
-CONC_EVENT *thisev = conc_allocate_event (EV_RECV);
-
- /* Fill in all of the fields of a receive-event structure. */
-thisev->s.recv.source = ncptl_virtual_to_physical (procmap, ivar_a_task);
-thisev->s.recv.size = var_len;
-thisev->s.recv.alignment = 0LL;
-thisev->s.recv.misaligned = 0;
-thisev->s.recv.touching = 0;
-thisev->s.recv.verification = 0;
-thisev->s.recv.tag = 0LL;
-thisev->s.recv.pendingsends = pendingsends;
-thisev->s.recv.pendingrecvs = pendingrecvs;
-thisev->s.recv.bufferofs = 0LL;
-thisev->s.recv.buffernum = thisev->s.recv.verification ? pendingsends+pendingrecvs : 0;
-(void) ncptl_malloc_message (thisev->s.recv.size+thisev->s.recv.bufferofs, thisev->s.recv.alignment, thisev->s.recv.buffernum, thisev->s.recv.misaligned);
-thisev->s.recv.buffer = NULL;
-thisev->s.recv.tag = map_tag_into_MPI_range (thisev->s.recv.tag);
-}
-}
-}
-}
- /* THEN... */
- /* TASKS var_n SUCH THAT ((var_n)>(0LL)) && ((var_n)<(var_task_count)) SENDS TO TASK ncptl_func_modulo((var_n)+(1LL),var_task_count) */
-{
-ncptl_int var_n = virtrank;
-if (((var_n)>(0LL)) && ((var_n)<(var_task_count))) {   /* TASKS var_n SUCH THAT ((var_n)>(0LL)) && ((var_n)<(var_task_count)) */
- /* The current coNCePTuaL statement applies to our task. */
-ncptl_int virtdest = ncptl_func_modulo((var_n)+(1LL),var_task_count);
-if (virtdest>=0 && virtdest<var_num_tasks) {
- /* In this scope, ncptl_func_modulo((var_n)+(1LL),var_task_count) represents a single receiver. */
- /* Prepare to send 1LL messages. */
-ncptl_int numreps = 1LL;   /* Number of messages */
-if (numreps > 1LL) {
-CONC_EVENT * repeatev;   /* Event specifying the number of repetitions to perform */
-repeatev = conc_allocate_event (EV_REPEAT);
-repeatev->s.rep.end_event = ncptl_queue_length(eventqueue);
-repeatev->s.rep.numreps = numreps;
-}
-
- /* Ensure we have at least one message to send. */
-if (numreps > 0LL) {
-CONC_EVENT *thisev = conc_allocate_event (EV_SEND);
-
- /* Fill in all of the fields of a send-event structure. */
-thisev->s.send.dest = ncptl_virtual_to_physical (procmap, ncptl_func_modulo((var_n)+(1LL),var_task_count));
-thisev->s.send.size = var_len;
-thisev->s.send.alignment = 0LL;
-thisev->s.send.misaligned = 0;
-thisev->s.send.touching = 0;
-thisev->s.send.verification = 0;
-thisev->s.send.tag = 0LL;
-thisev->s.send.pendingsends = pendingsends;
-thisev->s.send.pendingrecvs = pendingrecvs;
-thisev->s.send.bufferofs = 0LL;
-thisev->s.send.buffernum = thisev->s.send.verification ? pendingsends+pendingrecvs : 0;
-(void) ncptl_malloc_message (thisev->s.send.size+thisev->s.send.bufferofs, thisev->s.send.alignment, thisev->s.send.buffernum, thisev->s.send.misaligned);
-thisev->s.send.buffer = NULL;
-thisev->s.send.tag = map_tag_into_MPI_range (thisev->s.send.tag);
-}
-}
-}
-}
- /* THEN... */
- /* TASK 0LL LOGS "# of tasks" AND "Latency (usecs)" AND "Latency (usecs)" AND "Latency (usecs)" */
-if ((0LL) == virtrank) {   /* TASK 0LL */
- /* The current coNCePTuaL statement applies to our task. */
-CONC_EVENT *thisev = conc_allocate_event (EV_CODE);
-thisev->s.code.number = 1;
-thisev->s.code.procmap = NULL;
-thisev->s.code.var_task_count = var_task_count;
-}
-}
-}
-
- /* Assign the number of events to repeat, now that we know that number. */
-if (!unroll_loop && numreps > 1LL) {
-repevent = repeventnum + (CONC_EVENT *) ncptl_queue_contents (eventqueue, 0);
-repevent->s.rep.end_event = ncptl_queue_length (eventqueue) - 1;
-}
-
- /* After the warmups are finished, synchronize all tasks. */
-if (loopnum == 0LL) {
-CONC_EVENT * thisev_sync;   /* Barrier-synchronization event */
-
-thisev_sync =  conc_allocate_event (EV_SYNC);
-thisev_sync->s.sync.communicator = MPI_COMM_WORLD;
-}
-
- /* After the warmups are finished, resume outputting. */
-if (numreps > 0LL && loopnum == 0LL) {
-CONC_EVENT * thisev_off;   /* Output un-suppression event */
-
-thisev_off =  conc_allocate_event (EV_SUPPRESS);
-thisev_off->s.suppress.quiet = 0;
-thisev_off->s.suppress.matching_event = on_event;
-}
-}
-}
- /* THEN... */
- /* TASK 0LL COMPUTES AGGREGATES */
-if ((0LL) == virtrank) {   /* TASK 0LL */
- /* The current coNCePTuaL statement applies to our task. */
-(void) conc_allocate_event (EV_FLUSH);
-}
-}
-}
-}
-}
-}
- /* Begin a new top-level statement. */
-(void) conc_allocate_event (EV_NEWSTMT);
-
- /*************************
-  * More boilerplate code *
-  *************************/
-
- /* Abort if the program will terminate with pending messages. */
-if (pendingsends && pendingrecvs)
-ncptl_fatal("Neglected to await the completion of %" NICS " asynchronous %s and %" NICS " asynchronous %s",
-pendingsends, pendingsends==1LL ? "send" : "sends",
-pendingrecvs, pendingrecvs==1LL ? "receive" : "receives");
-else
-if (pendingsends)
-ncptl_fatal("Neglected to await the completion of %" NICS " asynchronous %s",
-pendingsends, pendingsends==1LL ? "send" : "sends");
-else
-if (pendingrecvs)
-ncptl_fatal("Neglected to await the completion of %" NICS " asynchronous %s",
-pendingrecvs, pendingrecvs==1LL ? "receive" : "receives");
-
- /* Allocate memory for non-unique messages and asynchronous
-  * message handles now that we know how much memory we need
-  * to allocate. */
-eventlist = (CONC_EVENT *) ncptl_queue_contents (eventqueue, 0);
-numevents = ncptl_queue_length (eventqueue);
-sendrequests = (MPI_Request *) ncptl_queue_contents (sendreqQ, 0);
-recvrequests = (MPI_Request *) ncptl_queue_contents (recvreqQ, 0);
-for (i=0; i<numevents; i++) {
-CONC_EVENT *thisev = &eventlist[i];   /* Cache of the current event */
-switch (thisev->type) {
-case EV_SEND:
-if (!thisev->s.send.buffer)
-thisev->s.send.buffer = ncptl_malloc_message (thisev->s.send.bufferofs + thisev->s.send.size,
-thisev->s.send.alignment,
-thisev->s.send.buffernum,
-thisev->s.send.misaligned);
-if (thisev->s.send.verification)
-ncptl_fill_buffer (CONC_GETBUFPTR(send), thisev->s.send.bufferofs + thisev->s.send.size, -1);
-break;
-
-case EV_RECV:
-if (!thisev->s.recv.buffer)
-thisev->s.recv.buffer = ncptl_malloc_message (thisev->s.recv.bufferofs + thisev->s.recv.size,
-thisev->s.recv.alignment,
-thisev->s.recv.buffernum,
-thisev->s.recv.misaligned);
-if (thisev->s.recv.verification)
-ncptl_fill_buffer (CONC_GETBUFPTR(recv), thisev->s.recv.bufferofs + thisev->s.recv.size, -1);
-break;
-
-default:
-break;
-}
-}
-}
-
-/* Process a subset of the events in a given event list. */
-static void conc_process_events (CONC_EVENT *eventlist,
-ncptl_int firstev, ncptl_int lastev, ncptl_int numreps)
-{
-CONC_EVENT * thisev;   /* Cache of the current event */
-CONC_EVENT * thisev_first = &eventlist[firstev];   /* Cache of the first event */
-ncptl_int i;   /* Iterate over events. */
-ncptl_int j;   /* Iterate over repetitions. */
-
- /* Declarations specific to the c_mpi backend */
-MPI_Status status;   /* Not needed but required by MPI_Recv() */
-
- /* Process from event firstev to event lastev (both inclusive). */
-for (j=numreps; j>0; j--)
-for (i=firstev, thisev=thisev_first; i<=lastev; i++, thisev++) {
- /* Declare variables needed by all C-based backends. */
-
- /* Process a single event. */
-switch (thisev->type) {
-case EV_SEND:
- /* Synchronous send */
-(void) MPI_Send (CONC_GETBUFPTR(send),
-(int)thisev->s.send.size, MPI_BYTE,
-(int)thisev->s.send.dest, (int)thisev->s.send.tag, MPI_COMM_WORLD);
-break;
-
-case EV_RECV:
- /* Synchronous receive */
-(void) MPI_Recv (CONC_GETBUFPTR(recv),
-(int)thisev->s.recv.size, MPI_BYTE,
-(int)thisev->s.recv.source, (int)thisev->s.recv.tag,
-MPI_COMM_WORLD, &status);
-break;
-
-case EV_SYNC:
- /* Synchronize a subset of the tasks. */
-(void) MPI_Barrier (thisev->s.sync.communicator);
-break;
-
-case EV_RESET:
- /* Reset all of the counters exported to coNCePTuaL programs. */
-starttime = ncptl_time();
-break;
-
-case EV_FLUSH:
- /* Force all aggregate functions to produce a result. */
-if (!suppress_output) {
-uint64_t stop_elapsed_usecs = ncptl_time();
-var_elapsed_usecs = stop_elapsed_usecs - starttime;
-ncptl_log_compute_aggregates (logstate);
-starttime += ncptl_time() - stop_elapsed_usecs;
-}
-break;
-
-case EV_SUPPRESS:
- /* Turn logging and outputting on or off. */
-if (thisev->s.suppress.quiet) {
- /* Suppress output. */
-thisev->s.suppress.prev_quiet = suppress_output;
-suppress_output = 1;
-thisev->s.suppress.stop_elapsed_usecs = ncptl_time();
-}
-else {
- /* Restore suppression to its previous state. */
-CONC_SUPPRESS_EVENT * matchev = &eventlist[thisev->s.suppress.matching_event].s.suppress;   /* Matching suppression event */
-suppress_output = matchev->prev_quiet;
-if (!suppress_output) {
-starttime += ncptl_time() - matchev->stop_elapsed_usecs;
-}
-}
-break;
-
-case EV_REPEAT:
- /* Repeatedly perform the next batch of events. */
-conc_process_events (eventlist, i+1, thisev->s.rep.end_event, thisev->s.rep.numreps);
-i = thisev->s.rep.end_event;
-thisev = &eventlist[i];
-break;
-
-case EV_NEWSTMT:
- /* Begin a new table in the log file. */
-if (!suppress_output) {
-uint64_t stop_elapsed_usecs = ncptl_time();
-var_elapsed_usecs = stop_elapsed_usecs - starttime;
-ncptl_log_commit_data (logstate);
-starttime += ncptl_time() - stop_elapsed_usecs;
-}
-break;
-
-case EV_CODE:
- /* Execute an arbitrary piece of code. */
-switch (thisev->s.code.number) {
-case 0:
- /* TASK 0LL OUTPUTS ['"Performing "', 'var_trials', '" "', 'var_task_count', '"-task runs...."'] */
-if (!suppress_output) {
-uint64_t stop_elapsed_usecs = ncptl_time();
-var_elapsed_usecs = stop_elapsed_usecs - starttime;
-printf ("%s%.10lg%s%.10lg%s\n", "Performing ", (double)thisev->s.code.var_trials, " ", (double)thisev->s.code.var_task_count, "-task runs....");
-fflush (stdout);
-starttime += ncptl_time() - stop_elapsed_usecs;
-}
-break;
-
-case 1:
- /* TASK 0LL LOGS "# of tasks" AND "Latency (usecs)" AND "Latency (usecs)" AND "Latency (usecs)" */
-if (!suppress_output) {
-uint64_t stop_elapsed_usecs = ncptl_time();
-var_elapsed_usecs = stop_elapsed_usecs - starttime;
-ncptl_log_write (logstate, 0, "# of tasks", NCPTL_FUNC_ONLY, 0.0, (double)thisev->s.code.var_task_count);
-ncptl_log_write (logstate, 1, "Latency (usecs)", NCPTL_FUNC_MINIMUM, 0.0, ((double)var_elapsed_usecs)/((double)thisev->s.code.var_task_count));
-ncptl_log_write (logstate, 2, "Latency (usecs)", NCPTL_FUNC_MEAN, 0.0, ((double)var_elapsed_usecs)/((double)thisev->s.code.var_task_count));
-ncptl_log_write (logstate, 3, "Latency (usecs)", NCPTL_FUNC_VARIANCE, 0.0, ((double)var_elapsed_usecs)/((double)thisev->s.code.var_task_count));
-starttime += ncptl_time() - stop_elapsed_usecs;
-}
-break;
-
-default:
- /* The C code generation module must be broken. */
-ncptl_fatal ("Internal error: unknown EV_CODE block %" NICS, thisev->s.code.number);
-break;
-}
-break;
-
-default:
- /* The c_generic backend or the c_mpi backend must be broken. */
-ncptl_fatal ("Internal error: unknown event type %d", thisev->type);
-break;
-}
-}
-}
-
-/* Finish up cleanly and return a status code. */
-static int conc_finalize (void)
-{
-int exitcode = 0;   /* Program exit code (to pass to exit()) */
-
- /* Declarations specific to the c_mpi backend */
-int mpiresult;   /* Return code from MPI_Finalize() */
-char log_key_str[128];   /* String representing the range of valid MPI tags */
-
- /* Finalization code specific to the c_mpi backend */
-log_key_str[0] = '\0';
-if (conc_mcast_tallies[CONC_MCAST_MPI_BCAST] > 0) {
-char onefuncstr[50];
-sprintf (onefuncstr, "%sMPI_Bcast()*%" NICS,
-log_key_str[0] == '\0' ? "" : " ", conc_mcast_tallies[CONC_MCAST_MPI_BCAST]);
-strcat (log_key_str, onefuncstr);
-}
-if (conc_mcast_tallies[CONC_MCAST_MPI_ALLTOALL] > 0) {
-char onefuncstr[50];
-sprintf (onefuncstr, "%sMPI_Alltoall()*%" NICS,
-log_key_str[0] == '\0' ? "" : " ", conc_mcast_tallies[CONC_MCAST_MPI_ALLTOALL]);
-strcat (log_key_str, onefuncstr);
-}
-if (conc_mcast_tallies[CONC_MCAST_MPI_ALLTOALLV] > 0) {
-char onefuncstr[50];
-sprintf (onefuncstr, "%sMPI_Alltoallv()*%" NICS,
-log_key_str[0] == '\0' ? "" : " ", conc_mcast_tallies[CONC_MCAST_MPI_ALLTOALLV]);
-strcat (log_key_str, onefuncstr);
-}
-if (log_key_str[0] != '\0')
-ncptl_log_add_comment ("Multicast functions used (statically)", log_key_str);
-
- /* Write a standard epilogue to the log file. */
-ncptl_log_commit_data (logstate);
-ncptl_log_write_epilogue (logstate);
-ncptl_log_close (logstate);
-
- /* Inform the run-time library that it's no longer needed. */
-ncptl_queue_empty (eventqueue);
-ncptl_free (eventqueue);
-ncptl_finalize();
-
- /* Finalization code specific to the c_mpi backend */
-mpiresult = MPI_Finalize();
-mpi_is_running = 0;
-exitcode = mpiresult!=MPI_SUCCESS;
-
- /* Return an exit status code. */
-return exitcode;
-}
-
-/*************************************************************************/
-/***************************** MAIN ROUTINE ******************************/
-/*************************************************************************/
-
-/* Program execution starts here. */
-int main (int argc, char *argv[])
-{
- /* Declare variables needed by all C-based backends. */
-CONC_EVENT * eventlist;   /* List of events to execute */
-ncptl_int numevents;   /* Number of entries in eventlist[] */
-
- /* ----- Initialization ----- */
-conc_initialize (argc, argv);
-eventlist = (CONC_EVENT *) ncptl_queue_contents (eventqueue, 0);
-numevents = ncptl_queue_length (eventqueue);
-sendrequests = (MPI_Request *) ncptl_queue_contents (sendreqQ, 0);
-sendstatuses = (MPI_Status *) ncptl_queue_contents (sendstatQ, 0);
-recvrequests = (MPI_Request *) ncptl_queue_contents (recvreqQ, 0);
-recvstatuses = (MPI_Status *) ncptl_queue_contents (recvstatQ, 0);
-starttime = ncptl_time();
-
- /* ----- Event-list processing ----- */
-conc_process_events (eventlist, 0, numevents-1, 1);
-
- /* ----- Finalization ----- */
-return conc_finalize();
-}

From a5be58053b36d55be21f79d8fbdead2c07750f56 Mon Sep 17 00:00:00 2001
From: Xin Wang <xwang149@hawk.iit.edu>
Date: Tue, 22 Jan 2019 15:12:42 -0600
Subject: [PATCH 08/15] Delete hotpotato.ncptl

---
 scripts/conceptual_benchmarks/hotpotato.ncptl | 26 -------------------
 1 file changed, 26 deletions(-)
 delete mode 100644 scripts/conceptual_benchmarks/hotpotato.ncptl

diff --git a/scripts/conceptual_benchmarks/hotpotato.ncptl b/scripts/conceptual_benchmarks/hotpotato.ncptl
deleted file mode 100644
index c574bb18..00000000
--- a/scripts/conceptual_benchmarks/hotpotato.ncptl
+++ /dev/null
@@ -1,26 +0,0 @@
-# Virtual ring "hot potato" test
-
-Require language version "1.5".
-
-trials is "Number of trials to perform" and comes from "--trials" or
-  "-t" with default 100000.
-
-Assert that "the hot-potato test requires at least two tasks" with num_tasks>=2.
-
-Let len be 0 while {
-  for each task_count in {2, ..., num_tasks} {
-    task 0 outputs "Performing " and trials and " " and
-		   task_count and "-task runs...." then
-    for trials repetitions plus 5 warmup repetitions and a synchronization {
-      task 0 resets its counters then
-      task 0 sends a len byte message to unsuspecting task 1 then
-      task (n+1) mod task_count receives a len byte message from task n such that n<task_count then
-      task n such that n>0 /\ n<task_count sends a len byte message to unsuspecting task (n+1) mod task_count then
-      task 0 logs the task_count as "# of tasks" and
-		  the minimum of elapsed_usecs/task_count as "Latency (usecs)" and
-		  the mean of elapsed_usecs/task_count as "Latency (usecs)" and
-		  the variance of elapsed_usecs/task_count as "Latency (usecs)"
-    } then
-    task 0 computes aggregates
-  }
-}

From f8cff566981cc01a02330063705a271e220805a2 Mon Sep 17 00:00:00 2001
From: Xin Wang <xwang149@hawk.iit.edu>
Date: Tue, 22 Jan 2019 15:13:08 -0600
Subject: [PATCH 09/15] Delete latency-all.ncptl

---
 .../conceptual_benchmarks/latency-all.ncptl   | 23 -------------------
 1 file changed, 23 deletions(-)
 delete mode 100644 scripts/conceptual_benchmarks/latency-all.ncptl

diff --git a/scripts/conceptual_benchmarks/latency-all.ncptl b/scripts/conceptual_benchmarks/latency-all.ncptl
deleted file mode 100644
index a8dbd30a..00000000
--- a/scripts/conceptual_benchmarks/latency-all.ncptl
+++ /dev/null
@@ -1,23 +0,0 @@
-# An all-pairs ping-pong latency test written in coNCePTuaL
-# By Scott Pakin <pakin@lanl.gov>
-
-Require language version "1.5".
-
-# Parse the command line.
-reps is "Number of repetitions of each message size" and comes from "--reps" or "-r" with default 1000.
-maxbytes is "Maximum number of bytes to transmit" and comes from "--maxbytes" or "-m" with default 1M.
-
-# Ensure that we have a peer with whom to communicate.
-Assert that "the latency test requires at least two tasks" with num_tasks>=2.
-
-# Perform the benchmark.
-For each msgsize in {0}, {1, 2, 4, ..., maxbytes} {
-  for reps repetitions {
-    tasks ev such that ev is even reset their counters then
-    tasks ev such that ev is even send a msgsize byte message to task ev+1 then
-    tasks od such that od is odd send a msgsize byte message to task od-1 then
-    tasks ev such that ev is even log the msgsize as "Bytes" and
-                                      the median of elapsed_usecs/2 as "1/2 RTT (usecs)"
-  } then
-  tasks ev such that ev is even compute aggregates
-}

From f11bf89f1267cdea1e9dbc7f82da2eb952f801c5 Mon Sep 17 00:00:00 2001
From: Xin Wang <xwang149@hawk.iit.edu>
Date: Tue, 22 Jan 2019 15:13:56 -0600
Subject: [PATCH 10/15] Delete conc-bisect.c

---
 .../conceputal-skeleton-apps/conc-bisect.c    | 1285 -----------------
 1 file changed, 1285 deletions(-)
 delete mode 100644 src/workload/conceputal-skeleton-apps/conc-bisect.c

diff --git a/src/workload/conceputal-skeleton-apps/conc-bisect.c b/src/workload/conceputal-skeleton-apps/conc-bisect.c
deleted file mode 100644
index b16e1840..00000000
--- a/src/workload/conceputal-skeleton-apps/conc-bisect.c
+++ /dev/null
@@ -1,1285 +0,0 @@
-/**********************************************************************
- * This file was generated by coNCePTuaL on Mon Jan  7 23:10:02 2019
- * using the c_mpi backend (C + MPI).
- * Do not modify this file; modify /Users/xin/macworkspace/codes-dev/codes/scripts/conceptual_benchmarks/bisect.ncptl instead.
- *
- * Entire source program
- * ---------------------
- *   ###################################################
- *   # Measure random bisection-bandwidth patterns     #
- *   # By Scott Pakin <pakin@lanl.gov>                 #
- *   #                                                 #
- *   # Inspired by Hoefler, Schneider, and Lumsdaine's #
- *   # "Multistage Switches are not Crossbars" paper   #
- *   ###################################################
- *   
- *   Require language version "1.5".
- *   
- *   nummsgs is "Number of messages per trial" and comes from "--nummsgs" or "-n" with default 100.
- *   wups is "Number of warmup messages" and comes from "--wups" or "-w" with default 3.
- *   msgsize is "Message size in bytes" and comes from "--bytes" or "-b" with default 1M.
- *   numtrials is "Number of bisection patterns" and comes from "--trials" or "-t" with default 5000.
- *   rounding is "Round measurements to the nearest N" and comes from "--round" or "-r" with default 5.
- *   
- *   For each trial in {1, ..., numtrials} {
- *     task 0 is assigned to processor 0 then
- *     task 0 outputs "Testing random bisection pattern " and trial and "/" and numtrials then
- *     all tasks are assigned to a random processor then
- *     tasks src such that src is even send wups msgsize-byte messages to task src+1 then
- *     all tasks synchronize then
- *     all tasks reset their counters then
- *     tasks src such that src is even send nummsgs msgsize-byte messages to task src+1 then
- *     all tasks synchronize then
- *     all tasks log a histogram of rounding*round(total_bytes*1E6/(elapsed_usecs*1M)/rounding) as "Bandwidth (MiB/s)"
- *   }
- **********************************************************************/
-
-#include "codes/codes-conc-addon.h"
-/*****************
- * Include files *
- *****************/
-
-/* Header files needed by all C-based backends */
-#include <stdio.h>
-#include <string.h>
-#include <ncptl/ncptl.h>
-
-/* Header files specific to the c_mpi backend */
-#include <mpi.h>
-#include <stdarg.h>
-
-/**********
- * Macros *
- **********/
-
-/* Define the maximum loop trip count that we're willing to unroll fully. */
-#define CONC_MAX_UNROLL 5
-
-/* Specify the minimum number of trial iterations in each FOR <time> loop. */
-#define CONC_FOR_TIME_TRIALS 1
-
-/* Define a macro that rounds a double to a ncptl_int. */
-#define CONC_DBL2INT(D) ((ncptl_int)((D)+0.5))
-
-/* Define a macro that increments a buffer pointer by a byte offset. */
-#define CONC_GETBUFPTR(S) ((void *)((char *)thisev->s.S.buffer + thisev->s.S.bufferofs))
-
-/* Implement ncptl_func_task_of in terms of ncptl_physical_to_virtual. */
-#define ncptl_func_task_of(P) ((ncptl_int)(P) < 0LL || (ncptl_int)(P) >= var_num_tasks ? -1LL : ncptl_physical_to_virtual (procmap, (ncptl_int)(P)))
-#define ncptl_dfunc_task_of(P) ((double) ncptl_func_task_of(P))
-
-/* Implement ncptl_func_processor_of in terms of ncptl_virtual_to_physical. */
-#define ncptl_func_processor_of(V) ((ncptl_int)(V) < 0LL || (ncptl_int)(V) >= var_num_tasks ? -1LL : ncptl_virtual_to_physical (procmap, (ncptl_int)(V)))
-#define ncptl_dfunc_processor_of(V) ((double) ncptl_func_processor_of(V))
-
-/* Define a macro that increments REDUCE's alternate buffer pointer by a byte offset. */
-#define CONC_GETALTBUFPTR(S) ((void *)((char *)thisev->s.S.altbuffer + thisev->s.S.bufferofs))
-
-/* Estimate the number of unique communicators that this program will need.
- * (The tradeoff is one of initialization time versus memory consumption.) */
-#define ESTIMATED_COMMUNICATORS 128
-
-/* Specify an operation to use for all reduction operations. */
-#define REDUCE_OPERATION MPI_SUM
-#define REDUCE_OPERATION_NAME "MPI_SUM"
-
-
-/*********************
- * Type declarations *
- *********************/
-
-/* Enumerate the various mechanisms used to implement MULTICAST statements. */
-typedef enum {
-CONC_MCAST_MPI_BCAST,       /* One to many */
-CONC_MCAST_MPI_ALLTOALL,    /* Many to many, same data to all */ 
-CONC_MCAST_MPI_ALLTOALLV,   /* General many to many */
-CONC_MCAST_MPI_NUM_FUNCS    /* Number of the above */
-} CONC_MCAST_MPI_FUNC;
-
-/* Define the type of event to perform. */
-typedef enum {
-EV_SEND,     /* Synchronous send */
-EV_ASEND,    /* Asynchronous send */
-EV_RECV,     /* Synchronous receive */
-EV_ARECV,    /* Asynchronous receive */
-EV_WAIT,     /* Wait for all asynchronous sends/receives to complete */
-EV_DELAY,    /* Spin or sleep */
-EV_TOUCH,    /* Touch a region of memory */
-EV_SYNC,     /* Barrier synchronization */
-EV_RESET,    /* Reset counters */
-EV_STORE,    /* Store all counters' current values */
-EV_RESTORE,  /* Restore the previously pushed counter values */
-EV_FLUSH,    /* Compute aggregate functions for log-file columns */
-EV_MCAST,    /* Synchronous multicast */
-EV_REDUCE,   /* Reduction with or without a subsequent multicast */
-EV_BTIME,    /* Beginning of a timed loop */
-EV_ETIME,    /* Ending of a timed loop */
-EV_REPEAT,   /* Repeatedly process the next N events */
-EV_SUPPRESS, /* Suppress writing to the log and standard output */
-EV_NEWSTMT,  /* Beginning of a new top-level statement */
-EV_CODE,     /* None of the above */
-NUM_EVS      /* Number of event types in CONC_EVENT_TYPE */
-} CONC_EVENT_TYPE;
-
-/* Describe a synchronous or asynchronous send event. */
-typedef struct {
-ncptl_int dest;         /* Destination task */
-ncptl_int size;         /* Number of bytes to send */
-ncptl_int alignment;    /* Message alignment (in bytes) */
-ncptl_int pendingsends; /* # of outstanding sends */
-ncptl_int pendingrecvs; /* # of outstanding receives */
-ncptl_int buffernum;    /* Buffer # to send from */
-ncptl_int bufferofs;    /* Byte offset into the message buffer */
-ncptl_int tag;          /* Tag to use for selective receives */
-int misaligned;         /* 1=misaligned from a page; 0=align as specified */
-int touching;           /* 1=touch every word before sending */
-int verification;       /* 1=fill message buffer with known contents */
-void *buffer;           /* Pointer to message memory */
-MPI_Request * handle;   /* MPI handle representing an asynchronous send */
-} CONC_SEND_EVENT;
-
-/* Describe a synchronous or asynchronous receive event. */
-typedef struct {
-ncptl_int source;       /* Source task */
-ncptl_int size;         /* Number of bytes to receive */
-ncptl_int alignment;    /* Message alignment (in bytes) */
-ncptl_int pendingsends; /* # of outstanding sends */
-ncptl_int pendingrecvs; /* # of outstanding receives */
-ncptl_int buffernum;    /* Buffer # to receive into */
-ncptl_int bufferofs;    /* Byte offset into the message buffer */
-ncptl_int tag;          /* Tag to use for selective receives */
-int misaligned;         /* 1=misaligned from a page; 0=align as specified */
-int touching;           /* 1=touch every word after reception */
-int verification;       /* 1=verify that all bits are correct */
-void *buffer;           /* Pointer to message memory */
-MPI_Request * handle;   /* MPI handle representing an asynchronous receive */
-} CONC_RECV_EVENT;
-
-/* Describe a wait-for-asynchronous-completions event. */
-typedef struct {
-ncptl_int numsends;     /* # of sends we expect to complete. */
-ncptl_int numrecvs;     /* # of receives we expect to complete. */
-ncptl_int numrecvbytes; /* # of bytes we expect to receive-complete */
-ncptl_int *touchedlist;    /* List of receives that need to be touched */
-ncptl_int numtouches;         /* # of elements in the above */
-} CONC_WAIT_EVENT;
-
-/* Describe a spin or sleep delay. */
-typedef struct {
-uint64_t microseconds;  /* Length of delay in microseconds */
-int spin0sleep1;        /* 0=spin; 1=sleep */
-} CONC_DELAY_EVENT;
-
-/* Describe a barrier synchronization event. */
-typedef struct {
-MPI_Comm communicator;   /* Set of tasks to synchronize */
-} CONC_SYNC_EVENT;
-
-/* Describe a walk over a memory-region. */
-typedef struct {
-ncptl_int regionbytes;  /* Size in bytes of the region to touch */
-ncptl_int bytestride;   /* Stride in bytes to touch */
-ncptl_int numaccesses;  /* Number of words to touch */
-ncptl_int wordsize;     /* Size in bytes of each touch */
-ncptl_int firstbyte;    /* Byte offset of the first byte to touch */
-} CONC_TOUCH_EVENT;
-
-/* Describe a synchronous multicast event. */
-typedef struct {
-ncptl_int source;       /* Source task, -1 in the many-to-many case */
-ncptl_int size;         /* Number of bytes to send */
-ncptl_int alignment;    /* Message alignment (in bytes) */
-ncptl_int pendingsends; /* # of outstanding sends */
-ncptl_int pendingrecvs; /* # of outstanding receives */
-ncptl_int buffernum;    /* Buffer # to send/receive from */
-ncptl_int bufferofs;    /* Byte offset into the message buffer */
-ncptl_int tag;          /* Tag to use for selective receives */
-int misaligned;         /* 1=misaligned from a page; 0=align as specified */
-int touching;           /* 1=touch every word before sending */
-int verification;       /* 1=verify that all bits are correct */
-void *buffer;           /* Pointer to message memory */
-ncptl_int size2;   /* Number of bytes to receive in the many-to-many case */
-ncptl_int bufferofs2;   /* Byte offset into the message buffer in the many-to-many case */
-void * buffer2;   /* Pointer to receive-message memory in the many-to-many case */
-MPI_Comm communicator;   /* Set of tasks to multicast to/from */
-int root;   /* source's rank within communicator */
-int * sndvol;   /* Volume of data to send to each rank in the communicator */
-int * snddisp;   /* Offset from buffer of each message to send */
-int * rcvvol;   /* Volume of data to receive from each rank in the communicator */
-int * rcvdisp;   /* Offset from buffer2 of each message to receive */
-CONC_MCAST_MPI_FUNC mpi_func;   /* MPI function to use to perform the multicast */
-} CONC_MCAST_EVENT;
-
-/* Describe a reduction event. */
-typedef struct {
-ncptl_int numitems;     /* # of items to reduce */
-ncptl_int itemsize;     /* # of bytes per item */
-ncptl_int alignment;    /* Message alignment (in bytes) */
-ncptl_int pendingsends; /* # of outstanding sends */
-ncptl_int pendingrecvs; /* # of outstanding receives */
-ncptl_int buffernum;    /* Buffer # to send/receive from */
-ncptl_int bufferofs;    /* Byte offset into the message buffer */
-ncptl_int tag;          /* Tag to use for selective receives */
-int misaligned;         /* 1=misaligned from a page; 0=align as specified */
-int touching;           /* 1=touch every word before sending/after receiving */
-int sending;            /* 1=we're a sender */
-int receiving;          /* 1=we're a receiver */
-void *buffer;           /* Pointer to message memory */
-void * altbuffer;   /* Pointer to additional message memory */
-MPI_Comm sendcomm;   /* Set of tasks to reduce from */
-MPI_Comm recvcomm;   /* Set of tasks to reduce to */
-MPI_Datatype datatype;   /* MPI datatype to reduce */
-int reducetype;   /* 0=reduce; 1=allreduce; 2=reduce+bcast */
-ncptl_int reduceroot;   /* Root task of the reduction if reducetype is 0 or 2 */
-ncptl_int bcastroot;   /* Root task of the multicast if reducetype is 2 */
-} CONC_REDUCE_EVENT;
-
-/* Describe an event representing the beginning of a timed loop. */
-typedef struct {
-uint64_t usecs;         /* Requested loop duration */
-uint64_t warmup_usecs;  /* Requested duration of warmup loops */
-uint64_t starttime;     /* Time at which the loop state last changed */
-uint64_t itersleft;     /* # of iterations remaining */
-uint64_t previters;     /* # of iterations we performed last time */
-int prev_quiet;         /* Previous value of suppress_output */
-int timing_trial;       /* 1=performing a timing trial; 0=running for real */
-volatile int finished;  /* 1=time has expired; 0=still ticking */
-ncptl_int var_total_bytes;   /* Cached copy of var_total_bytes to restore after the trial runs */
-} CONC_BTIME_EVENT;
-
-/* Describe an event representing the end of a timed loop. */
-typedef struct {
-ncptl_int begin_event;  /* Index into eventlist[] of the corresponding BTIME event */
-} CONC_ETIME_EVENT;
-
-/* Describe an event representing repetitions of subsequent events. */
-typedef struct {
-ncptl_int end_event;    /* Index into eventlist[] of the last event to repeat */
-ncptl_int numreps;      /* # of repetitions to perform */
-} CONC_REPEAT_EVENT;
-
-/* Describe an event representing output suppression (either on or off). */
-typedef struct conc_suppress_event {
-int quiet;              /* 0=allow output; 1=suppress it */
-int prev_quiet;         /* Previous value of suppress_output */
-ncptl_int matching_event;  /* Event ID of the "suppression on" event */
-ncptl_int var_total_bytes;   /* Cached copy of var_total_bytes to restore after re-enabling output */
-uint64_t stop_elapsed_usecs;   /* Time at which we suppressed output */
-} CONC_SUPPRESS_EVENT;
-
-/* Describe an event representing arbitrary code to execute at run time. */
-typedef struct {
-ncptl_int number;       /* Unique number corresponding to a specific piece of code */NCPTL_VIRT_PHYS_MAP *procmap;  /* Current mapping between tasks and processors */
-ncptl_int var_numtrials;   /* Copy of var_numtrials to use within a piece of code */
-ncptl_int var_trial;   /* Copy of var_trial to use within a piece of code */
-ncptl_int var_rounding;   /* Copy of var_rounding to use within a piece of code */
-} CONC_CODE_EVENT;
-
-/* Describe an arbitrary coNCePTuaL event. */
-typedef struct {
-CONC_EVENT_TYPE type;          /* Type of event */
-union {
-CONC_SEND_EVENT send;          /* Send state */
-CONC_RECV_EVENT recv;          /* Receive state */
-CONC_WAIT_EVENT wait;          /* Wait-for-completions state */
-CONC_DELAY_EVENT delay;        /* State for spins and sleeps */
-CONC_TOUCH_EVENT touch;        /* State for memory touching */
-CONC_SYNC_EVENT sync;          /* Synchronization state */
-CONC_MCAST_EVENT mcast;        /* Multicast state */
-CONC_REDUCE_EVENT reduce;      /* Reduction state */
-CONC_BTIME_EVENT btime;        /* Timed-loop state */
-CONC_ETIME_EVENT etime;        /* Additional timed-loop state */
-CONC_REPEAT_EVENT rep;         /* Repeated-events state */
-CONC_SUPPRESS_EVENT suppress;  /* State for suppressing output */
-CONC_CODE_EVENT code;          /* State for arbitrary code */
-} s;
-} CONC_EVENT;
-
-/* Fully specify an arbitrary for() loop (used by FOR EACH). */
-typedef struct {
-NCPTL_QUEUE *list_comp;  /* NULL=ordinary list; other=list comprehension values */
-int integral;        /* 1=integral values; 0=floating-point values */
-enum {               /* Comparison of loop variable to end variable */
-CONC_LEQ,                /* Increasing progression */
-CONC_GEQ                 /* Decreasing progression */
-} comparator;
-enum {               /* How to increment the loop variable */
-CONC_ADD,                /* Arithmetically */
-CONC_MULT,               /* Geometrically increasing */
-CONC_DIV                 /* Geometrically decreasing */
-} increment;
-union {
-struct {
-ncptl_int loopvar;   /* Loop variable */
-ncptl_int prev_loopvar; /* Previous value of loop variable */
-ncptl_int startval;  /* Initial value of loop variable */
-ncptl_int endval;    /* Value not to exceed */
-ncptl_int incval;    /* Loop-variable increment */
-} i;
-struct {
-double loopvar;      /* Loop variable */
-double prev_loopvar; /* Previous value of loop variable */
-double startval;     /* Initial value of loop variable */
-double endval;       /* Value not to exceed */
-double incval;       /* Loop-variable increment */
-} d;
-} u;
-} LOOPBOUNDS;
-
-/********************
- * Global variables *
- ********************/
-
-/* Variables exported to coNCePTuaL programs */
-static ncptl_int var_bytes_received = 0;   /* Total number of bytes received */
-static ncptl_int var_msgs_received = 0;   /* Total number of messages received */
-static ncptl_int var_bit_errors = 0;   /* Total number of bit errors observed */
-static ncptl_int var_total_msgs = 0;   /* Sum of messages sent and messages received */
-static ncptl_int var_msgs_sent = 0;   /* Total number of messages sent */
-static ncptl_int var_bytes_sent = 0;   /* Total number of bytes sent */
-static ncptl_int var_num_tasks = 1;   /* Number of tasks running the program */
-static ncptl_int var_elapsed_usecs = 0;   /* Elapsed time in microseconds */
-static ncptl_int var_total_bytes = 0;   /* Sum of bytes sent and bytes received */
-
-/* Dummy variable to help mark other variables as used */
-static union {
-ncptl_int ni;
-int i;
-void *vp;
-} conc_dummy_var;
-
-/* Variables used internally by boilerplate code */
-static uint64_t starttime;   /* Time the clock was last reset (microseconds) */
-static ncptl_int pendingrecvs = 0;   /* Current # of outstanding receives */
-static ncptl_int pendingrecvbytes = 0; /* Current # of bytes in outstanding receives */
-static NCPTL_QUEUE *touchedqueue;      /* Queue of asynchronous receives to touch */
-static ncptl_int pendingsends = 0;   /* Current # of outstanding sends */
-static NCPTL_QUEUE *eventqueue;   /* List of coNCePTuaL events to perform */
-static int within_time_loop = 0;   /* 1=we're within a FOR <time> loop */
-static int suppress_output = 0;    /* 1=suppress output to stdout and the log file */
-static void *touch_region = NULL;   /* Memory region to touch */
-static ncptl_int touch_region_size = 0;   /* # of bytes in the above */
-static int virtrank;    /* This task's virtual rank in the computation */
-static int physrank;    /* This task's physical rank in the computation */
-static NCPTL_VIRT_PHYS_MAP *procmap;  /* Virtual to physical rank mapping */
-static ncptl_int random_seed;   /* Seed for the random-number generator */
-static NCPTL_LOG_FILE_STATE *logstate;   /* Opaque object representing all log-file state */
-static char *logfile_uuid;   /* Execution UUID to write to every log file */
-static char *logfiletmpl;   /* Template for the log file's name */
-//static char *logfiletmpl_default;   /* Default value of the above */
-
-/* Global variables specific to the c_mpi backend */
-static ncptl_int mpi_is_running = 0;   /* 1=MPI has been initialized */
-static NCPTL_QUEUE * recvreqQ;   /* List of MPI receive requests */
-static MPI_Request * recvrequests;   /* List version of recvreqQ */
-static NCPTL_QUEUE * recvstatQ;   /* List of MPI receive statuses */
-static MPI_Status * recvstatuses;   /* List version of recvstatQ */
-static NCPTL_QUEUE * sendreqQ;   /* List of MPI send requests */
-static MPI_Request * sendrequests;   /* List version of sendreqQ */
-static NCPTL_QUEUE * sendstatQ;   /* List of MPI send statuses */
-static MPI_Status * sendstatuses;   /* List version of sendstatQ */
-static NCPTL_SET * communicators;   /* Map from an array of processor flags to an MPI communicator */
-static MPI_Errhandler mpi_error_handler;   /* Handle to handle_MPI_error() */
-static ncptl_int mpi_tag_ub;   /* Upper bound on an MPI tag value */
-static ncptl_int conc_mcast_tallies[CONC_MCAST_MPI_NUM_FUNCS] = {0};   /* Tallies of (static) multicast implementation functions */
-
-/* Program-specific variables */
-static ncptl_int var_nummsgs;   /* Number of messages per trial (command-line argument) */
-static ncptl_int var_wups;   /* Number of warmup messages (command-line argument) */
-static ncptl_int var_msgsize;   /* Message size in bytes (command-line argument) */
-static ncptl_int var_numtrials;   /* Number of bisection patterns (command-line argument) */
-static ncptl_int var_rounding;   /* Round measurements to the nearest N (command-line argument) */
-
-/*************************
- * Function declarations *
- *************************/
-
-/* Make MPI errors invoke ncptl_fatal(). */
-static void handle_MPI_error (MPI_Comm *comm, int *errcode, ...)
-{
-va_list args;
-char errstring[MPI_MAX_ERROR_STRING];
-int errstrlen;
-
-va_start (args, errcode);
-if (MPI_Error_string (*errcode, errstring, &errstrlen) == MPI_SUCCESS)
-ncptl_fatal ("MPI run-time error: %s", errstring);
-else
-ncptl_fatal ("MPI aborted with unrecognized error code %d", *errcode);
-conc_dummy_var.vp = (void *) comm;   /* Prevent the compiler from complaining that comm is unused. */
-va_end (args);
-}
-
-/* Perform the equivalent of CODES_MPI_Comm_rank() for an arbitrary process. */
-static int rank_in_MPI_communicator (MPI_Comm subcomm, int global_rank)
-{
-  MPI_Group world_group;   /* Group associated with MPI_COMM_WORLD */
-  MPI_Group subgroup;      /* Group associate with subcomm */
-  int subrank;             /* global_rank's rank within subcomm */
-
-  MPI_Comm_group (MPI_COMM_WORLD, &world_group);
-  MPI_Comm_group (subcomm, &subgroup);
-  MPI_Group_translate_ranks (world_group, 1, &global_rank, subgroup, &subrank);
-  return subrank;
-}
-
-/* Map an arbitrary tag to within MPI's valid range of [0, mpi_tag_ub]. */
-static ncptl_int map_tag_into_MPI_range (ncptl_int tag)
-{
-if (tag == NCPTL_INT_MIN)
- /* Avoid taking the absolute value of NCPTL_INT_MIN. */
-tag = 555666773LL;   /* Arbitrary value */
-tag = ncptl_func_abs (tag);   /* Only nonnegatives values are allowed. */
-if (mpi_tag_ub < NCPTL_INT_MAX)
-tag %= mpi_tag_ub + 1;
-return tag;
-}
-
-/* Given an array of task in/out booleans return an MPI
- * communicator that represents the "in" tasks. */
-static MPI_Comm define_MPI_communicator (char *procflags)
-{
-MPI_Comm *existing_comm;    /* Previously defined MPI communicator */
-MPI_Comm new_comm;          /* Newly defined MPI communicator */
-
-existing_comm = (MPI_Comm *) ncptl_set_find (communicators, (void *)procflags);
-if (existing_comm)
-return *existing_comm;
-(void) MPI_Comm_split (MPI_COMM_WORLD, (int)procflags[physrank], physrank, &new_comm);
-//(void) MPI_Errhandler_set (new_comm, mpi_error_handler);
-ncptl_set_insert (communicators, (void *)procflags, (void *)&new_comm);
-return define_MPI_communicator (procflags);
-}
-
-/* Return 1 if a sequence loop will take at least one trip. */
-static int conc_seq_nonempty (LOOPBOUNDS *seq)
-{
-ncptl_int startval;   /* Integer version of seq's startval element */
-ncptl_int endval;   /* Integer version of seq's endval element */
-
-if (seq->integral) {
-startval = seq->u.i.startval;
-endval = seq->u.i.endval;
-}
-else {
-startval = CONC_DBL2INT (seq->u.d.startval);
-endval = CONC_DBL2INT (seq->u.d.endval);
-}
-switch (seq->comparator) {
-case CONC_LEQ:
-return startval <= endval;
-
-case CONC_GEQ:
-return startval >= endval;
-
-default:
-ncptl_fatal ("Internal error -- unknown comparator");
-}
-return -1;     /* Appease idiotic compilers. */
-}
-
-/* Initialize a sequence loop. */
-static void conc_seq_init (LOOPBOUNDS *seq)
-{
-if (seq->integral) {
-seq->u.i.loopvar = seq->u.i.startval;
-seq->u.i.prev_loopvar = seq->u.i.loopvar - 1;
-}
-else {
-seq->u.d.loopvar = seq->u.d.startval;
-seq->u.d.prev_loopvar = seq->u.d.loopvar - 1.0;
-}
-}
-
-/* Return 1 if a sequence loop should continue, 0 when finished. */
-static int conc_seq_continue (LOOPBOUNDS *seq)
-{
-LOOPBOUNDS seq_int;   /* Integer equivalent of *seq */
-if (seq->integral)
-seq_int = *seq;
-else {
-seq_int.u.i.loopvar = CONC_DBL2INT (seq->u.d.loopvar);
-seq_int.u.i.prev_loopvar = CONC_DBL2INT (seq->u.d.prev_loopvar);
-seq_int.u.i.endval = CONC_DBL2INT (seq->u.d.endval);
-}
-
-if (seq_int.u.i.loopvar == seq_int.u.i.prev_loopvar)
-return 0;
-switch (seq->comparator) {
-case CONC_LEQ:
-return seq_int.u.i.loopvar <= seq_int.u.i.endval;
-
-case CONC_GEQ:
-return seq_int.u.i.loopvar >= seq_int.u.i.endval;
-
-default:
-ncptl_fatal ("Internal error -- unknown comparator");
-}
-return -1;     /* Appease idiotic compilers. */
-}
-
-/* Proceed to the next iteration of a sequence loop. */
-static void conc_seq_next (LOOPBOUNDS *seq)
-{
-if (seq->integral) {
-seq->u.i.prev_loopvar = seq->u.i.loopvar;
-switch (seq->increment) {
-case CONC_ADD:
-seq->u.i.loopvar += seq->u.i.incval;
-break;
-
-case CONC_MULT:
-seq->u.i.loopvar *= seq->u.i.incval;
-break;
-
-case CONC_DIV:
-seq->u.i.loopvar /= seq->u.i.incval;
-break;
-
-default:
-ncptl_fatal ("Internal error -- unknown incrementer");
-}
-}
-else {
-seq->u.d.prev_loopvar = seq->u.d.loopvar;
-switch (seq->increment) {
-case CONC_ADD:
-seq->u.d.loopvar += seq->u.d.incval;
-break;
-
-case CONC_MULT:
-seq->u.d.loopvar *= seq->u.d.incval;
-break;
-
-case CONC_DIV:
-seq->u.d.loopvar /= seq->u.d.incval;
-break;
-
-default:
-ncptl_fatal ("Internal error -- unknown incrementer");
-}
-}
-}
-
-/* Inhibit the compiler from complaining that
- * certain variables are defined but not used.
- * This function should never be called. */
-static void conc_mark_variables_used (void)
-{
-conc_dummy_var.ni = var_bytes_received;
-conc_dummy_var.ni = var_msgs_received;
-conc_dummy_var.ni = var_bit_errors;
-conc_dummy_var.ni = var_total_msgs;
-conc_dummy_var.ni = var_msgs_sent;
-conc_dummy_var.ni = var_bytes_sent;
-conc_dummy_var.ni = var_num_tasks;
-conc_dummy_var.ni = var_elapsed_usecs;
-conc_dummy_var.ni = var_total_bytes;
-conc_dummy_var.ni = pendingrecvbytes;
-conc_dummy_var.ni = touch_region_size;
-conc_dummy_var.vp = touch_region;
-conc_dummy_var.i  = within_time_loop;
-conc_dummy_var.i  = suppress_output;
-rank_in_MPI_communicator (MPI_COMM_WORLD, 0);
-}
-
-/* Allocate a new event of a given type and return a pointer to it. */
-static CONC_EVENT *conc_allocate_event (CONC_EVENT_TYPE type)
-{
-CONC_EVENT *newevent = (CONC_EVENT *) ncptl_queue_allocate (eventqueue);
-
-newevent->type = type;
-return newevent;
-}
-
-/* Declare an exit handler that gets called automatically when the
- * program terminates, whether successfully or not. */
-static void conc_exit_handler (void)
-{
-if (mpi_is_running)
-MPI_Abort (MPI_COMM_WORLD, 1);
-}
-
-
-/* Initialize coNCePTuaL, the messaging layer, and this program itself. */
-static void conc_initialize (int argc, char *argv[])
-{
- /* Variables needed by all C-based backends */
-CONC_EVENT * eventlist;   /* List of events to execute */
-ncptl_int numevents;   /* Number of entries in eventlist[] */
-int help_only = 0;   /* 1=User specified --help; save time by skipping ncptl_init() */
-char * argv0 = strrchr(argv[0], '/') ? strrchr(argv[0], '/')+1 : argv[0];   /* Base name of the executable program */
-int i;   /* Generic loop variable */
-
- /* Declare all of our command-line arguments. */
-NCPTL_CMDLINE arguments[] = {
-{ NCPTL_TYPE_INT, NULL, "seed", 'S', "Seed for the random-number generator", {0}},
-{ NCPTL_TYPE_STRING, NULL, "logfile", 'L', "Log-file template", {0}},
-{ NCPTL_TYPE_INT, NULL, "nummsgs", 'n', "Number of messages per trial", {0}},
-{ NCPTL_TYPE_INT, NULL, "wups", 'w', "Number of warmup messages", {0}},
-{ NCPTL_TYPE_INT, NULL, "bytes", 'b', "Message size in bytes", {0}},
-{ NCPTL_TYPE_INT, NULL, "trials", 't', "Number of bisection patterns", {0}},
-{ NCPTL_TYPE_INT, NULL, "round", 'r', "Round measurements to the nearest N", {0}}
-};
-
- /* Incorporate the complete coNCePTuaL source code as an array
-//  * for use by ncptl_log_write_prologue(). */
-char *sourcecode[] = {
-"###################################################",
-"# Measure random bisection-bandwidth patterns     #",
-"# By Scott Pakin <pakin@lanl.gov>                 #",
-"#                                                 #",
-"# Inspired by Hoefler, Schneider, and Lumsdaine's #",
-"# \"Multistage Switches are not Crossbars\" paper   #",
-"###################################################",
-"",
-"Require language version \"1.5\".",
-"",
-"nummsgs is \"Number of messages per trial\" and comes from \"--nummsgs\" or \"-n\" with default 100.",
-"wups is \"Number of warmup messages\" and comes from \"--wups\" or \"-w\" with default 3.",
-"msgsize is \"Message size in bytes\" and comes from \"--bytes\" or \"-b\" with default 1M.",
-"numtrials is \"Number of bisection patterns\" and comes from \"--trials\" or \"-t\" with default 5000.",
-"rounding is \"Round measurements to the nearest N\" and comes from \"--round\" or \"-r\" with default 5.",
-"",
-"For each trial in {1, ..., numtrials} {",
-"  task 0 is assigned to processor 0 then",
-"  task 0 outputs \"Testing random bisection pattern \" and trial and \"/\" and numtrials then",
-"  all tasks are assigned to a random processor then",
-"  tasks src such that src is even send wups msgsize-byte messages to task src+1 then",
-"  all tasks synchronize then",
-"  all tasks reset their counters then",
-"  tasks src such that src is even send nummsgs msgsize-byte messages to task src+1 then",
-"  all tasks synchronize then",
-"  all tasks log a histogram of rounding*round(total_bytes*1E6/(elapsed_usecs*1M)/rounding) as \"Bandwidth (MiB/s)\"",
-"}",
-NULL
-};
-
- /* Variables specific to the c_mpi backend */
-int num_tasks;   /* int version of var_num_tasks needed by CODES_MPI_Comm_size() */
-char * procflags;   /* Array of 1s representing an all-task MPI communicator */
-MPI_Comm comm_world = MPI_COMM_WORLD;   /* Copy of MPI_COMM_WORLD that we can take the address of */
-void * attr_val;   /* Pointed to the value of MPI_TAG_UB */
-int attr_flag = 0;   /* true=MPI_TAG_UB was extracted; false=not extracted */
-char log_key_str[128];   /* String representing the range of valid MPI tags */
-
- /* As a special case, if the command line contains --help, then skip
-  * the coNCePTuaL initialization step. */
-for (i=1; i<argc; i++)
-if (!strcmp(argv[i], "--"))
-break;
-else
-if (!strcmp(argv[i], "--help") || !strcmp(argv[i], "-?")) {
-argv[1] = "-?";   /* Guaranteed to work, even with getopt() */
-help_only = 1;
-break;
-}
-
- /* Perform various initializations specific to the c_mpi backend. */
- /* Initialize MPI. */
-//(void) MPI_Init(&argc, &argv);
-mpi_is_running = 1;
-
- /* Initialize the coNCePTuaL run-time library. */
-if (!help_only)
-//ncptl_init (NCPTL_RUN_TIME_VERSION, argv[0]);
-//(void) atexit (conc_exit_handler);
-
- /* Initialize the communication routines needed by the c_mpi backend. */
-//(void) MPI_Errhandler_create ((MPI_Handler_function *)handle_MPI_error, &mpi_error_handler);
-//(void) MPI_Errhandler_set (MPI_COMM_WORLD, mpi_error_handler);
-(void) CODES_MPI_Comm_rank(MPI_COMM_WORLD, &physrank);
-(void) CODES_MPI_Comm_size(MPI_COMM_WORLD, &num_tasks);
-var_num_tasks = (ncptl_int) num_tasks;
-//(void) MPI_Comm_get_attr(MPI_COMM_WORLD, MPI_TAG_UB, &attr_val, &attr_flag);
-mpi_tag_ub = (ncptl_int) (attr_flag ? *(int *)attr_val : 32767);
-
- /* Seed the random-task-number generator. */
-random_seed = ncptl_seed_random_task(random_seed, 0);
-arguments[0].defaultvalue.intval = random_seed;
-
- /* Generate and broadcast a UUID. */
-//logfile_uuid = ncptl_log_generate_uuid();
-//(void) CODES_MPI_Bcast ((void *)logfile_uuid, 37, MPI_CHAR, 0, MPI_COMM_WORLD);
-
- /* Plug variables and default values into the NCPTL_CMDLINE structure. */
-arguments[0].variable = (CMDLINE_VALUE *) &random_seed;
-arguments[0].defaultvalue.intval = random_seed;
-arguments[1].variable = (CMDLINE_VALUE *) &logfiletmpl;
-arguments[2].variable = (CMDLINE_VALUE *) &var_nummsgs;
-arguments[2].defaultvalue.intval = 100LL;
-arguments[3].variable = (CMDLINE_VALUE *) &var_wups;
-arguments[3].defaultvalue.intval = 3LL;
-arguments[4].variable = (CMDLINE_VALUE *) &var_msgsize;
-arguments[4].defaultvalue.intval = 1048576LL;
-arguments[5].variable = (CMDLINE_VALUE *) &var_numtrials;
-arguments[5].defaultvalue.intval = 5000LL;
-arguments[6].variable = (CMDLINE_VALUE *) &var_rounding;
-arguments[6].defaultvalue.intval = 5LL;
-//logfiletmpl_default = (char *) ncptl_malloc (strlen(argv0) + 15, 0);
-//sprintf (logfiletmpl_default, "%s-%%p.log", argv0);
-//arguments[1].defaultvalue.stringval = logfiletmpl_default;
-
- /* Parse the command line. */
-mpi_is_running = 0;   /* Don't invoke MPI_Abort() after --help. */
-ncptl_parse_command_line (argc, argv, arguments, sizeof(arguments)/sizeof(NCPTL_CMDLINE));
-if (help_only)
-ncptl_fatal ("Internal error in the c_generic backend: failed to exit after giving help");
-mpi_is_running = 1;
-
- /* Broadcast the random-task-number generator's seed. */
-{
-int rndseed_int = (int) random_seed;   /* Version of random_seed with int type */
-(void) CODES_MPI_Bcast ((void *)&rndseed_int, 1, MPI_INT, 0, MPI_COMM_WORLD);
-random_seed = (ncptl_int) rndseed_int;
-}
-(void) ncptl_seed_random_task(random_seed, (ncptl_int)physrank);
-
- /* Establish a mapping from (virtual) task IDs to (physical) ranks. */
-procmap = ncptl_allocate_task_map (var_num_tasks);
-virtrank = ncptl_physical_to_virtual (procmap, physrank);
-
- /* Perform initializations specific to the c_mpi backend. */
-//ncptl_log_add_comment ("MPI send routines", "MPI_Send() and CODES_MPI_Isend()");
-//ncptl_log_add_comment ("MPI reduction operation", REDUCE_OPERATION_NAME);
-//sprintf (log_key_str, "[0, %" NICS "]", mpi_tag_ub);
-//ncptl_log_add_comment ("MPI tag range", log_key_str);
-//
-// /* Open the log file and write some standard prologue information to it. */
-//logstate = ncptl_log_open (logfiletmpl, physrank);
-////ncptl_log_write_prologue (logstate, argv[0], logfile_uuid, "c_mpi", "C + MPI",
-//var_num_tasks,
-//arguments, sizeof(arguments)/sizeof(NCPTL_CMDLINE),
-//sourcecode);
-//ncptl_free (logfile_uuid);
-
- /* Allocate a variety of dynamically growing queues. */
-eventqueue = ncptl_queue_init (sizeof (CONC_EVENT));
-touchedqueue = ncptl_queue_init (sizeof (ncptl_int));
-
- /* Perform initializations specific to the c_mpi backend. */
-sendreqQ = ncptl_queue_init (sizeof (MPI_Request));
-sendstatQ = ncptl_queue_init (sizeof (MPI_Status));
-recvreqQ = ncptl_queue_init (sizeof (MPI_Request));
-recvstatQ = ncptl_queue_init (sizeof (MPI_Status));
-communicators = ncptl_set_init (ESTIMATED_COMMUNICATORS, var_num_tasks*sizeof(char), sizeof(MPI_Comm));procflags = (char *) ncptl_malloc (var_num_tasks*sizeof(char), 0);
-for (i=0; i<var_num_tasks; i++)
-procflags[i] = 1;
-ncptl_set_insert (communicators, (void *)procflags, (void *)&comm_world);
-ncptl_free (procflags);
-
- /****************************************************
-  * Generated, program-specific initialization code. *
-  ****************************************************/
-
- /* FOR EACH var_trial IN [(['1LL'], 'var_numtrials')]... */
-{
-LOOPBOUNDS loopbounds[1];   /* List of range descriptions */
-ncptl_int rangenum;   /* Current offset into loopbounds[] */
-ncptl_int initial_vals[1];   /* Cache of the initial, enumerated values */
-ncptl_int final_val;   /* Cache of the final value */
-
- /* Write range 0's loop bounds, "next" function, and termination function to loopbounds[0]. */
-loopbounds[0].list_comp = NULL;
-initial_vals[0] = 1LL;
-final_val = var_numtrials;
-loopbounds[0].integral = 1;
-loopbounds[0].u.i.startval = initial_vals[0];
-loopbounds[0].u.i.endval = final_val;
-loopbounds[0].comparator = initial_vals[0]<=final_val ? CONC_LEQ : CONC_GEQ;
-loopbounds[0].increment = CONC_ADD;
-loopbounds[0].u.i.incval = initial_vals[0]<=final_val ? 1LL : -1LL;
-
- /* Now that we've defined all of our ranges we iterate over each range
-  * and each element within each range. */
-for (rangenum=0LL; rangenum<1; rangenum++) {
-LOOPBOUNDS * thisrange = &loopbounds[rangenum];   /* Current range */
-if (conc_seq_nonempty (thisrange))
-for (conc_seq_init (thisrange);
-conc_seq_continue (thisrange);
-conc_seq_next (thisrange)) {
-ncptl_int var_trial;
-if (thisrange->list_comp == NULL)
- /* The loopvar field contains the value for var_trial. */
-var_trial = thisrange->integral ? thisrange->u.i.loopvar : CONC_DBL2INT(thisrange->u.d.loopvar);
-else
- /* The queue of list-comprehension values provides the value for var_trial. */
-var_trial = *(ncptl_int *)ncptl_queue_pop(thisrange->list_comp);
-{
- /* TASK 0LL ARE ASSIGNED PROCESSOR 0LL */
-{
- /* In the following, "task ID" implies a virtual rank in the
-  * computation while "processor" implies a physical rank. */
-ncptl_int virtID = 0LL;   /* Task ID */
-procmap = ncptl_conditionally_copy_task_map (procmap);
-virtrank = ncptl_assign_processor (virtID, 0LL, procmap, physrank);
-}
- /* THEN... */
- /* TASK 0LL OUTPUTS ['"Testing random bisection pattern "', 'var_trial', '"/"', 'var_numtrials'] */
-if ((0LL) == virtrank) {   /* TASK 0LL */
- /* The current coNCePTuaL statement applies to our task. */
-CONC_EVENT *thisev = conc_allocate_event (EV_CODE);
-thisev->s.code.number = 0;
-thisev->s.code.procmap = NULL;
-thisev->s.code.var_numtrials = var_numtrials;
-thisev->s.code.var_trial = var_trial;
-}
- /* THEN... */
- /* ALL TASKS ARE ASSIGNED PROCESSOR [random] */
-{
- /* In the following, "task ID" implies a virtual rank in the
-  * computation while "processor" implies a physical rank. */
-ncptl_int virtID;   /* Task ID */
-procmap = ncptl_conditionally_copy_task_map (procmap);
-for (virtID=0; virtID<var_num_tasks; virtID++) {
-virtrank = ncptl_assign_processor (virtID, ncptl_random_task(0, var_num_tasks-1, -1), procmap, physrank);
-}
-}
- /* THEN... */
- /* TASKS var_src SUCH THAT ((var_src)&1)==0 RECEIVES FROM TASK (var_src)+(1LL) */
-{
-ncptl_int var_src;
- /* Loop over all tasks to see which will send to us. */
-for (var_src=0; var_src<var_num_tasks; var_src++)
-if (((var_src)&1)==0) {
- /* var_src now represents one of the tasks that will send to us. */
-ncptl_int ivar_a_task = var_src;
-if (virtrank == ((var_src)+(1LL))) {
- /* In this scope, we must be a message recipient. */
- /* Prepare to receive var_wups messages. */
-ncptl_int numreps = var_wups;   /* Number of messages */
-if (numreps > 1LL) {
-CONC_EVENT * repeatev;   /* Event specifying the number of repetitions to perform */
-repeatev = conc_allocate_event (EV_REPEAT);
-repeatev->s.rep.end_event = ncptl_queue_length(eventqueue);
-repeatev->s.rep.numreps = numreps;
-}
-
- /* Ensure we have at least one message to receive. */
-if (numreps > 0LL) {
-CONC_EVENT *thisev = conc_allocate_event (EV_RECV);
-
- /* Fill in all of the fields of a receive-event structure. */
-thisev->s.recv.source = ncptl_virtual_to_physical (procmap, ivar_a_task);
-thisev->s.recv.size = var_msgsize;
-thisev->s.recv.alignment = 0LL;
-thisev->s.recv.misaligned = 0;
-thisev->s.recv.touching = 0;
-thisev->s.recv.verification = 0;
-thisev->s.recv.tag = 0LL;
-thisev->s.recv.pendingsends = pendingsends;
-thisev->s.recv.pendingrecvs = pendingrecvs;
-thisev->s.recv.bufferofs = 0LL;
-thisev->s.recv.buffernum = thisev->s.recv.verification ? pendingsends+pendingrecvs : 0;
-(void) ncptl_malloc_message (thisev->s.recv.size+thisev->s.recv.bufferofs, thisev->s.recv.alignment, thisev->s.recv.buffernum, thisev->s.recv.misaligned);
-thisev->s.recv.buffer = NULL;
-thisev->s.recv.tag = map_tag_into_MPI_range (thisev->s.recv.tag);
-}
-}
-}
-}
- /* TASKS var_src SUCH THAT ((var_src)&1)==0 SENDS TO TASK (var_src)+(1LL) */
-{
-ncptl_int var_src = virtrank;
-if (((var_src)&1)==0) {   /* TASKS var_src SUCH THAT ((var_src)&1)==0 */
- /* The current coNCePTuaL statement applies to our task. */
-ncptl_int virtdest = (var_src)+(1LL);
-if (virtdest>=0 && virtdest<var_num_tasks) {
- /* In this scope, (var_src)+(1LL) represents a single receiver. */
- /* Prepare to send var_wups messages. */
-ncptl_int numreps = var_wups;   /* Number of messages */
-if (numreps > 1LL) {
-CONC_EVENT * repeatev;   /* Event specifying the number of repetitions to perform */
-repeatev = conc_allocate_event (EV_REPEAT);
-repeatev->s.rep.end_event = ncptl_queue_length(eventqueue);
-repeatev->s.rep.numreps = numreps;
-}
-
- /* Ensure we have at least one message to send. */
-if (numreps > 0LL) {
-CONC_EVENT *thisev = conc_allocate_event (EV_SEND);
-if (virtrank == ((var_src)+(1LL)))
-ncptl_fatal ("Send-to-self deadlock encountered on task %d in line 21 of the source code", virtrank);
-
- /* Fill in all of the fields of a send-event structure. */
-thisev->s.send.dest = ncptl_virtual_to_physical (procmap, (var_src)+(1LL));
-thisev->s.send.size = var_msgsize;
-thisev->s.send.alignment = 0LL;
-thisev->s.send.misaligned = 0;
-thisev->s.send.touching = 0;
-thisev->s.send.verification = 0;
-thisev->s.send.tag = 0LL;
-thisev->s.send.pendingsends = pendingsends;
-thisev->s.send.pendingrecvs = pendingrecvs;
-thisev->s.send.bufferofs = 0LL;
-thisev->s.send.buffernum = thisev->s.send.verification ? pendingsends+pendingrecvs : 0;
-(void) ncptl_malloc_message (thisev->s.send.size+thisev->s.send.bufferofs, thisev->s.send.alignment, thisev->s.send.buffernum, thisev->s.send.misaligned);
-thisev->s.send.buffer = NULL;
-thisev->s.send.tag = map_tag_into_MPI_range (thisev->s.send.tag);
-}
-}
-}
-}
- /* THEN... */
- /* ALL TASKS SYNCHRONIZE */
-{   /* ALL TASKS */
- /* The current coNCePTuaL statement applies to our task. */
-CONC_EVENT *thisev = conc_allocate_event (EV_SYNC);
-thisev->s.sync.communicator = MPI_COMM_WORLD;
-}
- /* THEN... */
- /* ALL TASKS RESET THEIR COUNTERS */
-{   /* ALL TASKS */
- /* The current coNCePTuaL statement applies to our task. */
-(void) conc_allocate_event (EV_RESET);
-}
- /* THEN... */
- /* TASKS var_src SUCH THAT ((var_src)&1)==0 RECEIVES FROM TASK (var_src)+(1LL) */
-{
-ncptl_int var_src;
- /* Loop over all tasks to see which will send to us. */
-for (var_src=0; var_src<var_num_tasks; var_src++)
-if (((var_src)&1)==0) {
- /* var_src now represents one of the tasks that will send to us. */
-ncptl_int ivar_b_task = var_src;
-if (virtrank == ((var_src)+(1LL))) {
- /* In this scope, we must be a message recipient. */
- /* Prepare to receive var_nummsgs messages. */
-ncptl_int numreps = var_nummsgs;   /* Number of messages */
-if (numreps > 1LL) {
-CONC_EVENT * repeatev;   /* Event specifying the number of repetitions to perform */
-repeatev = conc_allocate_event (EV_REPEAT);
-repeatev->s.rep.end_event = ncptl_queue_length(eventqueue);
-repeatev->s.rep.numreps = numreps;
-}
-
- /* Ensure we have at least one message to receive. */
-if (numreps > 0LL) {
-CONC_EVENT *thisev = conc_allocate_event (EV_RECV);
-
- /* Fill in all of the fields of a receive-event structure. */
-thisev->s.recv.source = ncptl_virtual_to_physical (procmap, ivar_b_task);
-thisev->s.recv.size = var_msgsize;
-thisev->s.recv.alignment = 0LL;
-thisev->s.recv.misaligned = 0;
-thisev->s.recv.touching = 0;
-thisev->s.recv.verification = 0;
-thisev->s.recv.tag = 0LL;
-thisev->s.recv.pendingsends = pendingsends;
-thisev->s.recv.pendingrecvs = pendingrecvs;
-thisev->s.recv.bufferofs = 0LL;
-thisev->s.recv.buffernum = thisev->s.recv.verification ? pendingsends+pendingrecvs : 0;
-(void) ncptl_malloc_message (thisev->s.recv.size+thisev->s.recv.bufferofs, thisev->s.recv.alignment, thisev->s.recv.buffernum, thisev->s.recv.misaligned);
-thisev->s.recv.buffer = NULL;
-thisev->s.recv.tag = map_tag_into_MPI_range (thisev->s.recv.tag);
-}
-}
-}
-}
- /* TASKS var_src SUCH THAT ((var_src)&1)==0 SENDS TO TASK (var_src)+(1LL) */
-{
-ncptl_int var_src = virtrank;
-if (((var_src)&1)==0) {   /* TASKS var_src SUCH THAT ((var_src)&1)==0 */
- /* The current coNCePTuaL statement applies to our task. */
-ncptl_int virtdest = (var_src)+(1LL);
-if (virtdest>=0 && virtdest<var_num_tasks) {
- /* In this scope, (var_src)+(1LL) represents a single receiver. */
- /* Prepare to send var_nummsgs messages. */
-ncptl_int numreps = var_nummsgs;   /* Number of messages */
-if (numreps > 1LL) {
-CONC_EVENT * repeatev;   /* Event specifying the number of repetitions to perform */
-repeatev = conc_allocate_event (EV_REPEAT);
-repeatev->s.rep.end_event = ncptl_queue_length(eventqueue);
-repeatev->s.rep.numreps = numreps;
-}
-
- /* Ensure we have at least one message to send. */
-if (numreps > 0LL) {
-CONC_EVENT *thisev = conc_allocate_event (EV_SEND);
-if (virtrank == ((var_src)+(1LL)))
-ncptl_fatal ("Send-to-self deadlock encountered on task %d in line 24 of the source code", virtrank);
-
- /* Fill in all of the fields of a send-event structure. */
-thisev->s.send.dest = ncptl_virtual_to_physical (procmap, (var_src)+(1LL));
-thisev->s.send.size = var_msgsize;
-thisev->s.send.alignment = 0LL;
-thisev->s.send.misaligned = 0;
-thisev->s.send.touching = 0;
-thisev->s.send.verification = 0;
-thisev->s.send.tag = 0LL;
-thisev->s.send.pendingsends = pendingsends;
-thisev->s.send.pendingrecvs = pendingrecvs;
-thisev->s.send.bufferofs = 0LL;
-thisev->s.send.buffernum = thisev->s.send.verification ? pendingsends+pendingrecvs : 0;
-(void) ncptl_malloc_message (thisev->s.send.size+thisev->s.send.bufferofs, thisev->s.send.alignment, thisev->s.send.buffernum, thisev->s.send.misaligned);
-thisev->s.send.buffer = NULL;
-thisev->s.send.tag = map_tag_into_MPI_range (thisev->s.send.tag);
-}
-}
-}
-}
- /* THEN... */
- /* ALL TASKS SYNCHRONIZE */
-{   /* ALL TASKS */
- /* The current coNCePTuaL statement applies to our task. */
-CONC_EVENT *thisev = conc_allocate_event (EV_SYNC);
-thisev->s.sync.communicator = MPI_COMM_WORLD;
-}
- /* THEN... */
- /* ALL TASKS LOGS "Bandwidth (MiB/s)" */
-{   /* ALL TASKS */
- /* The current coNCePTuaL statement applies to our task. */
-CONC_EVENT *thisev = conc_allocate_event (EV_CODE);
-thisev->s.code.number = 1;
-thisev->s.code.procmap = NULL;
-thisev->s.code.var_rounding = var_rounding;
-}
-}
-}
-}
-}
- /* Begin a new top-level statement. */
-(void) conc_allocate_event (EV_NEWSTMT);
-
- /*************************
-  * More boilerplate code *
-  *************************/
-
- /* Abort if the program will terminate with pending messages. */
-if (pendingsends && pendingrecvs)
-ncptl_fatal("Neglected to await the completion of %" NICS " asynchronous %s and %" NICS " asynchronous %s",
-pendingsends, pendingsends==1LL ? "send" : "sends",
-pendingrecvs, pendingrecvs==1LL ? "receive" : "receives");
-else
-if (pendingsends)
-ncptl_fatal("Neglected to await the completion of %" NICS " asynchronous %s",
-pendingsends, pendingsends==1LL ? "send" : "sends");
-else
-if (pendingrecvs)
-ncptl_fatal("Neglected to await the completion of %" NICS " asynchronous %s",
-pendingrecvs, pendingrecvs==1LL ? "receive" : "receives");
-
- /* Allocate memory for non-unique messages and asynchronous
-  * message handles now that we know how much memory we need
-  * to allocate. */
-eventlist = (CONC_EVENT *) ncptl_queue_contents (eventqueue, 0);
-numevents = ncptl_queue_length (eventqueue);
-sendrequests = (MPI_Request *) ncptl_queue_contents (sendreqQ, 0);
-recvrequests = (MPI_Request *) ncptl_queue_contents (recvreqQ, 0);
-for (i=0; i<numevents; i++) {
-CONC_EVENT *thisev = &eventlist[i];   /* Cache of the current event */
-switch (thisev->type) {
-case EV_SEND:
-if (!thisev->s.send.buffer)
-thisev->s.send.buffer = ncptl_malloc_message (thisev->s.send.bufferofs + thisev->s.send.size,
-thisev->s.send.alignment,
-thisev->s.send.buffernum,
-thisev->s.send.misaligned);
-if (thisev->s.send.verification)
-ncptl_fill_buffer (CONC_GETBUFPTR(send), thisev->s.send.bufferofs + thisev->s.send.size, -1);
-break;
-
-case EV_RECV:
-if (!thisev->s.recv.buffer)
-thisev->s.recv.buffer = ncptl_malloc_message (thisev->s.recv.bufferofs + thisev->s.recv.size,
-thisev->s.recv.alignment,
-thisev->s.recv.buffernum,
-thisev->s.recv.misaligned);
-if (thisev->s.recv.verification)
-ncptl_fill_buffer (CONC_GETBUFPTR(recv), thisev->s.recv.bufferofs + thisev->s.recv.size, -1);
-break;
-
-default:
-break;
-}
-}
-}
-
-/* Process a subset of the events in a given event list. */
-static void conc_process_events (CONC_EVENT *eventlist,
-ncptl_int firstev, ncptl_int lastev, ncptl_int numreps)
-{
-CONC_EVENT * thisev;   /* Cache of the current event */
-CONC_EVENT * thisev_first = &eventlist[firstev];   /* Cache of the first event */
-ncptl_int i;   /* Iterate over events. */
-ncptl_int j;   /* Iterate over repetitions. */
-
- /* Declarations specific to the c_mpi backend */
-MPI_Status status;   /* Not needed but required by CODES_MPI_Recv() */
-
- /* Process from event firstev to event lastev (both inclusive). */
-for (j=numreps; j>0; j--)
-for (i=firstev, thisev=thisev_first; i<=lastev; i++, thisev++) {
- /* Declare variables needed by all C-based backends. */
-
- /* Process a single event. */
-switch (thisev->type) {
-case EV_SEND:
- /* Synchronous send */
-(void) CODES_MPI_Send (CONC_GETBUFPTR(send),
-(int)thisev->s.send.size, MPI_BYTE,
-(int)thisev->s.send.dest, (int)thisev->s.send.tag, MPI_COMM_WORLD);
-var_total_bytes += thisev->s.send.size;
-break;
-
-case EV_RECV:
- /* Synchronous receive */
-(void) CODES_MPI_Recv (CONC_GETBUFPTR(recv),
-(int)thisev->s.recv.size, MPI_BYTE,
-(int)thisev->s.recv.source, (int)thisev->s.recv.tag,
-MPI_COMM_WORLD, &status);
-var_total_bytes += thisev->s.recv.size;
-break;
-
-case EV_SYNC:
- /* Synchronize a subset of the tasks. */
-(void) CODES_MPI_Barrier (thisev->s.sync.communicator);
-break;
-
-case EV_RESET:
- /* Reset all of the counters exported to coNCePTuaL programs. */
-var_total_bytes = 0;
-starttime = ncptl_time();
-break;
-
-case EV_REPEAT:
- /* Repeatedly perform the next batch of events. */
-conc_process_events (eventlist, i+1, thisev->s.rep.end_event, thisev->s.rep.numreps);
-i = thisev->s.rep.end_event;
-thisev = &eventlist[i];
-break;
-
-case EV_NEWSTMT:
- /* Begin a new table in the log file. */
-if (!suppress_output) {
-uint64_t stop_elapsed_usecs = ncptl_time();
-var_elapsed_usecs = stop_elapsed_usecs - starttime;
-//ncptl_log_commit_data (logstate);
-starttime += ncptl_time() - stop_elapsed_usecs;
-}
-break;
-
-case EV_CODE:
- /* Execute an arbitrary piece of code. */
-switch (thisev->s.code.number) {
-case 0:
- /* TASK 0LL OUTPUTS ['"Testing random bisection pattern "', 'var_trial', '"/"', 'var_numtrials'] */
-if (!suppress_output) {
-uint64_t stop_elapsed_usecs = ncptl_time();
-var_elapsed_usecs = stop_elapsed_usecs - starttime;
-printf ("%s%.10lg%s%.10lg\n", "Testing random bisection pattern ", (double)thisev->s.code.var_trial, "/", (double)thisev->s.code.var_numtrials);
-fflush (stdout);
-starttime += ncptl_time() - stop_elapsed_usecs;
-}
-break;
-
-case 1:
- /* ALL TASKS LOGS "Bandwidth (MiB/s)" */
-if (!suppress_output) {
-uint64_t stop_elapsed_usecs = ncptl_time();
-var_elapsed_usecs = stop_elapsed_usecs - starttime;
-//ncptl_log_write (logstate, 0, "Bandwidth (MiB/s)", NCPTL_FUNC_HISTOGRAM, 0.0, ((double)thisev->s.code.var_rounding)*(ncptl_dfunc_round(((((double)var_total_bytes)*(1000000.0))/(((double)var_elapsed_usecs)*(1048576.0)))/((double)thisev->s.code.var_rounding))));
-starttime += ncptl_time() - stop_elapsed_usecs;
-}
-break;
-
-default:
- /* The C code generation module must be broken. */
-ncptl_fatal ("Internal error: unknown EV_CODE block %" NICS, thisev->s.code.number);
-break;
-}
-break;
-
-default:
- /* The c_generic backend or the c_mpi backend must be broken. */
-ncptl_fatal ("Internal error: unknown event type %d", thisev->type);
-break;
-}
-}
-}
-
-/* Finish up cleanly and return a status code. */
-static int conc_finalize (void)
-{
-int exitcode = 0;   /* Program exit code (to pass to exit()) */
-
- /* Declarations specific to the c_mpi backend */
-//int mpiresult;   /* Return code from MPI_Finalize() */
-//char log_key_str[128];   /* String representing the range of valid MPI tags */
-//
-// /* Finalization code specific to the c_mpi backend */
-//log_key_str[0] = '\0';
-//if (conc_mcast_tallies[CONC_MCAST_MPI_BCAST] > 0) {
-//char onefuncstr[50];
-//sprintf (onefuncstr, "%sCODES_MPI_Bcast()*%" NICS,
-//log_key_str[0] == '\0' ? "" : " ", conc_mcast_tallies[CONC_MCAST_MPI_BCAST]);
-//strcat (log_key_str, onefuncstr);
-//}
-//if (conc_mcast_tallies[CONC_MCAST_MPI_ALLTOALL] > 0) {
-//char onefuncstr[50];
-//sprintf (onefuncstr, "%sCODES_MPI_Alltoall()*%" NICS,
-//log_key_str[0] == '\0' ? "" : " ", conc_mcast_tallies[CONC_MCAST_MPI_ALLTOALL]);
-//strcat (log_key_str, onefuncstr);
-//}
-//if (conc_mcast_tallies[CONC_MCAST_MPI_ALLTOALLV] > 0) {
-//char onefuncstr[50];
-//sprintf (onefuncstr, "%sCODES_MPI_Alltoallv()*%" NICS,
-//log_key_str[0] == '\0' ? "" : " ", conc_mcast_tallies[CONC_MCAST_MPI_ALLTOALLV]);
-//strcat (log_key_str, onefuncstr);
-//}
-//if (log_key_str[0] != '\0')
-//ncptl_log_add_comment ("Multicast functions used (statically)", log_key_str);
-//
-// /* Write a standard epilogue to the log file. */
-////ncptl_log_commit_data (logstate);
-////ncptl_log_write_epilogue (logstate);
-//ncptl_log_close (logstate);
-
- /* Inform the run-time library that it's no longer needed. */
-//ncptl_queue_empty (eventqueue);
-//ncptl_free (eventqueue);
-//ncptl_finalize();
-
- /* Finalization code specific to the c_mpi backend */
-CODES_MPI_Finalize();
-mpi_is_running = 0;
-exitcode = 0;
-
- /* Return an exit status code. */
-return exitcode;
-}
-
-/*************************************************************************/
-/***************************** MAIN ROUTINE ******************************/
-/*************************************************************************/
-
-/* Program execution starts here. */
-static int bisect_main (int argc, char *argv[])
-{
- /* Declare variables needed by all C-based backends. */
-CONC_EVENT * eventlist;   /* List of events to execute */
-ncptl_int numevents;   /* Number of entries in eventlist[] */
-
- /* ----- Initialization ----- */
-conc_initialize (argc, argv);
-eventlist = (CONC_EVENT *) ncptl_queue_contents (eventqueue, 0);
-numevents = ncptl_queue_length (eventqueue);
-sendrequests = (MPI_Request *) ncptl_queue_contents (sendreqQ, 0);
-sendstatuses = (MPI_Status *) ncptl_queue_contents (sendstatQ, 0);
-recvrequests = (MPI_Request *) ncptl_queue_contents (recvreqQ, 0);
-recvstatuses = (MPI_Status *) ncptl_queue_contents (recvstatQ, 0);
-starttime = ncptl_time();
-
- /* ----- Event-list processing ----- */
-conc_process_events (eventlist, 0, numevents-1, 1);
-
- /* ----- Finalization ----- */
-return conc_finalize();
-}
-
-/* fill in function pointers for this method */
-struct codes_conceptual_bench bisect_bench = 
-{
-.program_name = "bisect",
-.conceptual_main = bisect_main,
-};

From 8ac60867f1819e08de505a6d51ef1d606e5a7c56 Mon Sep 17 00:00:00 2001
From: Xin Wang <xwang149@hawk.iit.edu>
Date: Tue, 22 Jan 2019 15:14:12 -0600
Subject: [PATCH 11/15] Delete conc-cosmoflow.c

---
 .../conceputal-skeleton-apps/conc-cosmoflow.c | 1325 -----------------
 1 file changed, 1325 deletions(-)
 delete mode 100644 src/workload/conceputal-skeleton-apps/conc-cosmoflow.c

diff --git a/src/workload/conceputal-skeleton-apps/conc-cosmoflow.c b/src/workload/conceputal-skeleton-apps/conc-cosmoflow.c
deleted file mode 100644
index 5fc2d8db..00000000
--- a/src/workload/conceputal-skeleton-apps/conc-cosmoflow.c
+++ /dev/null
@@ -1,1325 +0,0 @@
-/**********************************************************************
- * This file was generated by coNCePTuaL on Mon Jan  7 23:10:01 2019
- * using the c_mpi backend (C + MPI).
- * Do not modify this file; modify /Users/xin/macworkspace/codes-dev/codes/scripts/conceptual_benchmarks/cosmoflow.ncptl instead.
- *
- * Entire source program
- * ---------------------
- *   # Measure the performance of MPI_Allgather()
- *   # By Scott Pakin <pakin@lanl.gov>
- *   #
- *   # N.B. Requires the c_mpi backend.
- *   Require language version "1.5".
- *   # Parse the command line.
- *   numwords is "Message size (words)" and comes from "--msgsize" or "-s" with default 28825K.
- *   reps is "Number of repetitions" and comes from "--reps" or "-r" with default 100.
- *   computetime is "Computation time (ms)" and comes from "--compute" or "-c" with default 129.
- *   
- *   # Allocate a send buffer and a receive buffer.
- *   Task 0 multicasts a numwords*num_tasks word message from buffer 0 to all other tasks.
- *   Task 0 multicasts a numwords*num_tasks word message from buffer 1 to all other tasks.
- *   
- *   # Measure the performance of CODES_MPI_Allreduce().
- *   Task 0 resets its counters then
- *   for reps repetitions {
- *       all tasks COMPUTES FOR computetime MILLISECONDS then
- *       all tasks backend execute "
- *           CODES_MPI_Allreduce([MESSAGE BUFFER 0], [MESSAGE BUFFER 1], (int)" and numwords and ",
- *                   MPI_INT, MPI_SUM, MPI_COMM_WORLD);
- *           " then
- *       all tasks backend execute "
- *           CODES_MPI_Allreduce([MESSAGE BUFFER 0], [MESSAGE BUFFER 1], (int)" and numwords and ",
- *                   MPI_INT, MPI_SUM, MPI_COMM_WORLD);
- *           "
- *   } then
- *   task 0 logs elapsed_usecs/1000 as "Elapse time (ms)".
- **********************************************************************/
-
-#include "codes/codes-conc-addon.h"
-/*****************
- * Include files *
- *****************/
-
-/* Header files needed by all C-based backends */
-#include <stdio.h>
-#include <string.h>
-#include <ncptl/ncptl.h>
-
-/* Header files specific to the c_mpi backend */
-#include <mpi.h>
-#include <stdarg.h>
-
-/**********
- * Macros *
- **********/
-
-/* Define the maximum loop trip count that we're willing to unroll fully. */
-#define CONC_MAX_UNROLL 5
-
-/* Specify the minimum number of trial iterations in each FOR <time> loop. */
-#define CONC_FOR_TIME_TRIALS 1
-
-/* Define a macro that rounds a double to a ncptl_int. */
-#define CONC_DBL2INT(D) ((ncptl_int)((D)+0.5))
-
-/* Define a macro that increments a buffer pointer by a byte offset. */
-#define CONC_GETBUFPTR(S) ((void *)((char *)thisev->s.S.buffer + thisev->s.S.bufferofs))
-
-/* Implement ncptl_func_task_of in terms of ncptl_physical_to_virtual. */
-#define ncptl_func_task_of(P) ((ncptl_int)(P) < 0LL || (ncptl_int)(P) >= var_num_tasks ? -1LL : ncptl_physical_to_virtual (procmap, (ncptl_int)(P)))
-#define ncptl_dfunc_task_of(P) ((double) ncptl_func_task_of(P))
-
-/* Implement ncptl_func_processor_of in terms of ncptl_virtual_to_physical. */
-#define ncptl_func_processor_of(V) ((ncptl_int)(V) < 0LL || (ncptl_int)(V) >= var_num_tasks ? -1LL : ncptl_virtual_to_physical (procmap, (ncptl_int)(V)))
-#define ncptl_dfunc_processor_of(V) ((double) ncptl_func_processor_of(V))
-
-/* Define a macro that increments REDUCE's alternate buffer pointer by a byte offset. */
-#define CONC_GETALTBUFPTR(S) ((void *)((char *)thisev->s.S.altbuffer + thisev->s.S.bufferofs))
-
-/* Estimate the number of unique communicators that this program will need.
- * (The tradeoff is one of initialization time versus memory consumption.) */
-#define ESTIMATED_COMMUNICATORS 128
-
-/* Specify an operation to use for all reduction operations. */
-#define REDUCE_OPERATION MPI_SUM
-#define REDUCE_OPERATION_NAME "MPI_SUM"
-
-
-/*********************
- * Type declarations *
- *********************/
-
-/* Enumerate the various mechanisms used to implement MULTICAST statements. */
-typedef enum {
-CONC_MCAST_MPI_BCAST,       /* One to many */
-CONC_MCAST_MPI_ALLTOALL,    /* Many to many, same data to all */ 
-CONC_MCAST_MPI_ALLTOALLV,   /* General many to many */
-CONC_MCAST_MPI_NUM_FUNCS    /* Number of the above */
-} CONC_MCAST_MPI_FUNC;
-
-/* Define the type of event to perform. */
-typedef enum {
-EV_SEND,     /* Synchronous send */
-EV_ASEND,    /* Asynchronous send */
-EV_RECV,     /* Synchronous receive */
-EV_ARECV,    /* Asynchronous receive */
-EV_WAIT,     /* Wait for all asynchronous sends/receives to complete */
-EV_DELAY,    /* Spin or sleep */
-EV_TOUCH,    /* Touch a region of memory */
-EV_SYNC,     /* Barrier synchronization */
-EV_RESET,    /* Reset counters */
-EV_STORE,    /* Store all counters' current values */
-EV_RESTORE,  /* Restore the previously pushed counter values */
-EV_FLUSH,    /* Compute aggregate functions for log-file columns */
-EV_MCAST,    /* Synchronous multicast */
-EV_REDUCE,   /* Reduction with or without a subsequent multicast */
-EV_BTIME,    /* Beginning of a timed loop */
-EV_ETIME,    /* Ending of a timed loop */
-EV_REPEAT,   /* Repeatedly process the next N events */
-EV_SUPPRESS, /* Suppress writing to the log and standard output */
-EV_NEWSTMT,  /* Beginning of a new top-level statement */
-EV_CODE,     /* None of the above */
-NUM_EVS      /* Number of event types in CONC_EVENT_TYPE */
-} CONC_EVENT_TYPE;
-
-/* Describe a synchronous or asynchronous send event. */
-typedef struct {
-ncptl_int dest;         /* Destination task */
-ncptl_int size;         /* Number of bytes to send */
-ncptl_int alignment;    /* Message alignment (in bytes) */
-ncptl_int pendingsends; /* # of outstanding sends */
-ncptl_int pendingrecvs; /* # of outstanding receives */
-ncptl_int buffernum;    /* Buffer # to send from */
-ncptl_int bufferofs;    /* Byte offset into the message buffer */
-ncptl_int tag;          /* Tag to use for selective receives */
-int misaligned;         /* 1=misaligned from a page; 0=align as specified */
-int touching;           /* 1=touch every word before sending */
-int verification;       /* 1=fill message buffer with known contents */
-void *buffer;           /* Pointer to message memory */
-MPI_Request * handle;   /* MPI handle representing an asynchronous send */
-} CONC_SEND_EVENT;
-
-/* Describe a synchronous or asynchronous receive event. */
-typedef struct {
-ncptl_int source;       /* Source task */
-ncptl_int size;         /* Number of bytes to receive */
-ncptl_int alignment;    /* Message alignment (in bytes) */
-ncptl_int pendingsends; /* # of outstanding sends */
-ncptl_int pendingrecvs; /* # of outstanding receives */
-ncptl_int buffernum;    /* Buffer # to receive into */
-ncptl_int bufferofs;    /* Byte offset into the message buffer */
-ncptl_int tag;          /* Tag to use for selective receives */
-int misaligned;         /* 1=misaligned from a page; 0=align as specified */
-int touching;           /* 1=touch every word after reception */
-int verification;       /* 1=verify that all bits are correct */
-void *buffer;           /* Pointer to message memory */
-MPI_Request * handle;   /* MPI handle representing an asynchronous receive */
-} CONC_RECV_EVENT;
-
-/* Describe a wait-for-asynchronous-completions event. */
-typedef struct {
-ncptl_int numsends;     /* # of sends we expect to complete. */
-ncptl_int numrecvs;     /* # of receives we expect to complete. */
-ncptl_int numrecvbytes; /* # of bytes we expect to receive-complete */
-ncptl_int *touchedlist;    /* List of receives that need to be touched */
-ncptl_int numtouches;         /* # of elements in the above */
-} CONC_WAIT_EVENT;
-
-/* Describe a spin or sleep delay. */
-typedef struct {
-uint64_t microseconds;  /* Length of delay in microseconds */
-int spin0sleep1;        /* 0=spin; 1=sleep */
-} CONC_DELAY_EVENT;
-
-/* Describe a barrier synchronization event. */
-typedef struct {
-MPI_Comm communicator;   /* Set of tasks to synchronize */
-} CONC_SYNC_EVENT;
-
-/* Describe a walk over a memory-region. */
-typedef struct {
-ncptl_int regionbytes;  /* Size in bytes of the region to touch */
-ncptl_int bytestride;   /* Stride in bytes to touch */
-ncptl_int numaccesses;  /* Number of words to touch */
-ncptl_int wordsize;     /* Size in bytes of each touch */
-ncptl_int firstbyte;    /* Byte offset of the first byte to touch */
-} CONC_TOUCH_EVENT;
-
-/* Describe a synchronous multicast event. */
-typedef struct {
-ncptl_int source;       /* Source task, -1 in the many-to-many case */
-ncptl_int size;         /* Number of bytes to send */
-ncptl_int alignment;    /* Message alignment (in bytes) */
-ncptl_int pendingsends; /* # of outstanding sends */
-ncptl_int pendingrecvs; /* # of outstanding receives */
-ncptl_int buffernum;    /* Buffer # to send/receive from */
-ncptl_int bufferofs;    /* Byte offset into the message buffer */
-ncptl_int tag;          /* Tag to use for selective receives */
-int misaligned;         /* 1=misaligned from a page; 0=align as specified */
-int touching;           /* 1=touch every word before sending */
-int verification;       /* 1=verify that all bits are correct */
-void *buffer;           /* Pointer to message memory */
-ncptl_int size2;   /* Number of bytes to receive in the many-to-many case */
-ncptl_int bufferofs2;   /* Byte offset into the message buffer in the many-to-many case */
-void * buffer2;   /* Pointer to receive-message memory in the many-to-many case */
-MPI_Comm communicator;   /* Set of tasks to multicast to/from */
-int root;   /* source's rank within communicator */
-int * sndvol;   /* Volume of data to send to each rank in the communicator */
-int * snddisp;   /* Offset from buffer of each message to send */
-int * rcvvol;   /* Volume of data to receive from each rank in the communicator */
-int * rcvdisp;   /* Offset from buffer2 of each message to receive */
-CONC_MCAST_MPI_FUNC mpi_func;   /* MPI function to use to perform the multicast */
-} CONC_MCAST_EVENT;
-
-/* Describe a reduction event. */
-typedef struct {
-ncptl_int numitems;     /* # of items to reduce */
-ncptl_int itemsize;     /* # of bytes per item */
-ncptl_int alignment;    /* Message alignment (in bytes) */
-ncptl_int pendingsends; /* # of outstanding sends */
-ncptl_int pendingrecvs; /* # of outstanding receives */
-ncptl_int buffernum;    /* Buffer # to send/receive from */
-ncptl_int bufferofs;    /* Byte offset into the message buffer */
-ncptl_int tag;          /* Tag to use for selective receives */
-int misaligned;         /* 1=misaligned from a page; 0=align as specified */
-int touching;           /* 1=touch every word before sending/after receiving */
-int sending;            /* 1=we're a sender */
-int receiving;          /* 1=we're a receiver */
-void *buffer;           /* Pointer to message memory */
-void * altbuffer;   /* Pointer to additional message memory */
-MPI_Comm sendcomm;   /* Set of tasks to reduce from */
-MPI_Comm recvcomm;   /* Set of tasks to reduce to */
-MPI_Datatype datatype;   /* MPI datatype to reduce */
-int reducetype;   /* 0=reduce; 1=allreduce; 2=reduce+bcast */
-ncptl_int reduceroot;   /* Root task of the reduction if reducetype is 0 or 2 */
-ncptl_int bcastroot;   /* Root task of the multicast if reducetype is 2 */
-} CONC_REDUCE_EVENT;
-
-/* Describe an event representing the beginning of a timed loop. */
-typedef struct {
-uint64_t usecs;         /* Requested loop duration */
-uint64_t warmup_usecs;  /* Requested duration of warmup loops */
-uint64_t starttime;     /* Time at which the loop state last changed */
-uint64_t itersleft;     /* # of iterations remaining */
-uint64_t previters;     /* # of iterations we performed last time */
-int prev_quiet;         /* Previous value of suppress_output */
-int timing_trial;       /* 1=performing a timing trial; 0=running for real */
-volatile int finished;  /* 1=time has expired; 0=still ticking */
-} CONC_BTIME_EVENT;
-
-/* Describe an event representing the end of a timed loop. */
-typedef struct {
-ncptl_int begin_event;  /* Index into eventlist[] of the corresponding BTIME event */
-} CONC_ETIME_EVENT;
-
-/* Describe an event representing repetitions of subsequent events. */
-typedef struct {
-ncptl_int end_event;    /* Index into eventlist[] of the last event to repeat */
-ncptl_int numreps;      /* # of repetitions to perform */
-} CONC_REPEAT_EVENT;
-
-/* Describe an event representing output suppression (either on or off). */
-typedef struct conc_suppress_event {
-int quiet;              /* 0=allow output; 1=suppress it */
-int prev_quiet;         /* Previous value of suppress_output */
-ncptl_int matching_event;  /* Event ID of the "suppression on" event */
-uint64_t stop_elapsed_usecs;   /* Time at which we suppressed output */
-} CONC_SUPPRESS_EVENT;
-
-/* Describe an event representing arbitrary code to execute at run time. */
-typedef struct {
-ncptl_int number;       /* Unique number corresponding to a specific piece of code */NCPTL_VIRT_PHYS_MAP *procmap;  /* Current mapping between tasks and processors */
-ncptl_int var_numwords;   /* Copy of var_numwords to use within a piece of code */
-} CONC_CODE_EVENT;
-
-/* Describe an arbitrary coNCePTuaL event. */
-typedef struct {
-CONC_EVENT_TYPE type;          /* Type of event */
-union {
-CONC_SEND_EVENT send;          /* Send state */
-CONC_RECV_EVENT recv;          /* Receive state */
-CONC_WAIT_EVENT wait;          /* Wait-for-completions state */
-CONC_DELAY_EVENT delay;        /* State for spins and sleeps */
-CONC_TOUCH_EVENT touch;        /* State for memory touching */
-CONC_SYNC_EVENT sync;          /* Synchronization state */
-CONC_MCAST_EVENT mcast;        /* Multicast state */
-CONC_REDUCE_EVENT reduce;      /* Reduction state */
-CONC_BTIME_EVENT btime;        /* Timed-loop state */
-CONC_ETIME_EVENT etime;        /* Additional timed-loop state */
-CONC_REPEAT_EVENT rep;         /* Repeated-events state */
-CONC_SUPPRESS_EVENT suppress;  /* State for suppressing output */
-CONC_CODE_EVENT code;          /* State for arbitrary code */
-} s;
-} CONC_EVENT;
-
-/* Fully specify an arbitrary for() loop (used by FOR EACH). */
-typedef struct {
-NCPTL_QUEUE *list_comp;  /* NULL=ordinary list; other=list comprehension values */
-int integral;        /* 1=integral values; 0=floating-point values */
-enum {               /* Comparison of loop variable to end variable */
-CONC_LEQ,                /* Increasing progression */
-CONC_GEQ                 /* Decreasing progression */
-} comparator;
-enum {               /* How to increment the loop variable */
-CONC_ADD,                /* Arithmetically */
-CONC_MULT,               /* Geometrically increasing */
-CONC_DIV                 /* Geometrically decreasing */
-} increment;
-union {
-struct {
-ncptl_int loopvar;   /* Loop variable */
-ncptl_int prev_loopvar; /* Previous value of loop variable */
-ncptl_int startval;  /* Initial value of loop variable */
-ncptl_int endval;    /* Value not to exceed */
-ncptl_int incval;    /* Loop-variable increment */
-} i;
-struct {
-double loopvar;      /* Loop variable */
-double prev_loopvar; /* Previous value of loop variable */
-double startval;     /* Initial value of loop variable */
-double endval;       /* Value not to exceed */
-double incval;       /* Loop-variable increment */
-} d;
-} u;
-} LOOPBOUNDS;
-
-/********************
- * Global variables *
- ********************/
-
-/* Variables exported to coNCePTuaL programs */
-static ncptl_int var_bytes_received = 0;   /* Total number of bytes received */
-static ncptl_int var_msgs_received = 0;   /* Total number of messages received */
-static ncptl_int var_bit_errors = 0;   /* Total number of bit errors observed */
-static ncptl_int var_total_msgs = 0;   /* Sum of messages sent and messages received */
-static ncptl_int var_msgs_sent = 0;   /* Total number of messages sent */
-static ncptl_int var_bytes_sent = 0;   /* Total number of bytes sent */
-static ncptl_int var_num_tasks = 1;   /* Number of tasks running the program */
-static ncptl_int var_elapsed_usecs = 0;   /* Elapsed time in microseconds */
-static ncptl_int var_total_bytes = 0;   /* Sum of bytes sent and bytes received */
-
-/* Dummy variable to help mark other variables as used */
-static union {
-ncptl_int ni;
-int i;
-void *vp;
-} conc_dummy_var;
-
-/* Variables used internally by boilerplate code */
-static uint64_t starttime;   /* Time the clock was last reset (microseconds) */
-static ncptl_int pendingrecvs = 0;   /* Current # of outstanding receives */
-static ncptl_int pendingrecvbytes = 0; /* Current # of bytes in outstanding receives */
-static NCPTL_QUEUE *touchedqueue;      /* Queue of asynchronous receives to touch */
-static ncptl_int pendingsends = 0;   /* Current # of outstanding sends */
-static NCPTL_QUEUE *eventqueue;   /* List of coNCePTuaL events to perform */
-static int within_time_loop = 0;   /* 1=we're within a FOR <time> loop */
-static int suppress_output = 0;    /* 1=suppress output to stdout and the log file */
-static void *touch_region = NULL;   /* Memory region to touch */
-static ncptl_int touch_region_size = 0;   /* # of bytes in the above */
-static int virtrank;    /* This task's virtual rank in the computation */
-static int physrank;    /* This task's physical rank in the computation */
-static NCPTL_VIRT_PHYS_MAP *procmap;  /* Virtual to physical rank mapping */
-static NCPTL_LOG_FILE_STATE *logstate;   /* Opaque object representing all log-file state */
-static char *logfile_uuid;   /* Execution UUID to write to every log file */
-static char *logfiletmpl;   /* Template for the log file's name */
-//static char *logfiletmpl_default;   /* Default value of the above */
-
-/* Global variables specific to the c_mpi backend */
-static ncptl_int mpi_is_running = 0;   /* 1=MPI has been initialized */
-static NCPTL_QUEUE * recvreqQ;   /* List of MPI receive requests */
-static MPI_Request * recvrequests;   /* List version of recvreqQ */
-static NCPTL_QUEUE * recvstatQ;   /* List of MPI receive statuses */
-static MPI_Status * recvstatuses;   /* List version of recvstatQ */
-static NCPTL_QUEUE * sendreqQ;   /* List of MPI send requests */
-static MPI_Request * sendrequests;   /* List version of sendreqQ */
-static NCPTL_QUEUE * sendstatQ;   /* List of MPI send statuses */
-static MPI_Status * sendstatuses;   /* List version of sendstatQ */
-static NCPTL_SET * communicators;   /* Map from an array of processor flags to an MPI communicator */
-static MPI_Errhandler mpi_error_handler;   /* Handle to handle_MPI_error() */
-static ncptl_int mpi_tag_ub;   /* Upper bound on an MPI tag value */
-static ncptl_int conc_mcast_tallies[CONC_MCAST_MPI_NUM_FUNCS] = {0};   /* Tallies of (static) multicast implementation functions */
-
-/* Program-specific variables */
-static ncptl_int var_numwords;   /* Message size (words) (command-line argument) */
-static ncptl_int var_reps;   /* Number of repetitions (command-line argument) */
-static ncptl_int var_computetime;   /* Computation time (ms) (command-line argument) */
-
-/*************************
- * Function declarations *
- *************************/
-
-/* Make MPI errors invoke ncptl_fatal(). */
-static void handle_MPI_error (MPI_Comm *comm, int *errcode, ...)
-{
-va_list args;
-char errstring[MPI_MAX_ERROR_STRING];
-int errstrlen;
-
-va_start (args, errcode);
-if (MPI_Error_string (*errcode, errstring, &errstrlen) == MPI_SUCCESS)
-ncptl_fatal ("MPI run-time error: %s", errstring);
-else
-ncptl_fatal ("MPI aborted with unrecognized error code %d", *errcode);
-conc_dummy_var.vp = (void *) comm;   /* Prevent the compiler from complaining that comm is unused. */
-va_end (args);
-}
-
-/* Perform the equivalent of CODES_MPI_Comm_rank() for an arbitrary process. */
-static int rank_in_MPI_communicator (MPI_Comm subcomm, int global_rank)
-{
-  MPI_Group world_group;   /* Group associated with MPI_COMM_WORLD */
-  MPI_Group subgroup;      /* Group associate with subcomm */
-  int subrank;             /* global_rank's rank within subcomm */
-
-  MPI_Comm_group (MPI_COMM_WORLD, &world_group);
-  MPI_Comm_group (subcomm, &subgroup);
-  MPI_Group_translate_ranks (world_group, 1, &global_rank, subgroup, &subrank);
-  return subrank;
-}
-
-/* Map an arbitrary tag to within MPI's valid range of [0, mpi_tag_ub]. */
-static ncptl_int map_tag_into_MPI_range (ncptl_int tag)
-{
-if (tag == NCPTL_INT_MIN)
- /* Avoid taking the absolute value of NCPTL_INT_MIN. */
-tag = 555666773LL;   /* Arbitrary value */
-tag = ncptl_func_abs (tag);   /* Only nonnegatives values are allowed. */
-if (mpi_tag_ub < NCPTL_INT_MAX)
-tag %= mpi_tag_ub + 1;
-return tag;
-}
-
-/* Given an array of task in/out booleans return an MPI
- * communicator that represents the "in" tasks. */
-static MPI_Comm define_MPI_communicator (char *procflags)
-{
-MPI_Comm *existing_comm;    /* Previously defined MPI communicator */
-MPI_Comm new_comm;          /* Newly defined MPI communicator */
-
-existing_comm = (MPI_Comm *) ncptl_set_find (communicators, (void *)procflags);
-if (existing_comm)
-return *existing_comm;
-(void) MPI_Comm_split (MPI_COMM_WORLD, (int)procflags[physrank], physrank, &new_comm);
-//(void) MPI_Errhandler_set (new_comm, mpi_error_handler);
-ncptl_set_insert (communicators, (void *)procflags, (void *)&new_comm);
-return define_MPI_communicator (procflags);
-}
-
-
-/* Inhibit the compiler from complaining that
- * certain variables are defined but not used.
- * This function should never be called. */
-static void conc_mark_variables_used (void)
-{
-conc_dummy_var.ni = var_bytes_received;
-conc_dummy_var.ni = var_msgs_received;
-conc_dummy_var.ni = var_bit_errors;
-conc_dummy_var.ni = var_total_msgs;
-conc_dummy_var.ni = var_msgs_sent;
-conc_dummy_var.ni = var_bytes_sent;
-conc_dummy_var.ni = var_num_tasks;
-conc_dummy_var.ni = var_elapsed_usecs;
-conc_dummy_var.ni = var_total_bytes;
-conc_dummy_var.ni = pendingrecvbytes;
-conc_dummy_var.ni = touch_region_size;
-conc_dummy_var.vp = touch_region;
-conc_dummy_var.i  = within_time_loop;
-conc_dummy_var.i  = suppress_output;
-rank_in_MPI_communicator (MPI_COMM_WORLD, 0);
-}
-
-/* Allocate a new event of a given type and return a pointer to it. */
-static CONC_EVENT *conc_allocate_event (CONC_EVENT_TYPE type)
-{
-CONC_EVENT *newevent = (CONC_EVENT *) ncptl_queue_allocate (eventqueue);
-
-newevent->type = type;
-return newevent;
-}
-
-/* Declare an exit handler that gets called automatically when the
- * program terminates, whether successfully or not. */
-static void conc_exit_handler (void)
-{
-if (mpi_is_running)
-MPI_Abort (MPI_COMM_WORLD, 1);
-}
-
-
-/* Initialize coNCePTuaL, the messaging layer, and this program itself. */
-static void conc_initialize (int argc, char *argv[])
-{
- /* Variables needed by all C-based backends */
-CONC_EVENT * eventlist;   /* List of events to execute */
-ncptl_int numevents;   /* Number of entries in eventlist[] */
-int help_only = 0;   /* 1=User specified --help; save time by skipping ncptl_init() */
-char * argv0 = strrchr(argv[0], '/') ? strrchr(argv[0], '/')+1 : argv[0];   /* Base name of the executable program */
-int i;   /* Generic loop variable */
-
- /* Declare all of our command-line arguments. */
-NCPTL_CMDLINE arguments[] = {
-{ NCPTL_TYPE_STRING, NULL, "logfile", 'L', "Log-file template", {0}},
-{ NCPTL_TYPE_INT, NULL, "msgsize", 's', "Message size (words)", {0}},
-{ NCPTL_TYPE_INT, NULL, "reps", 'r', "Number of repetitions", {0}},
-{ NCPTL_TYPE_INT, NULL, "compute", 'c', "Computation time (ms)", {0}}
-};
-
- /* Incorporate the complete coNCePTuaL source code as an array
-//  * for use by ncptl_log_write_prologue(). */
-char *sourcecode[] = {
-"# Measure the performance of MPI_Allgather()",
-"# By Scott Pakin <pakin@lanl.gov>",
-"#",
-"# N.B. Requires the c_mpi backend.",
-"Require language version \"1.5\".",
-"# Parse the command line.",
-"numwords is \"Message size (words)\" and comes from \"--msgsize\" or \"-s\" with default 28825K.",
-"reps is \"Number of repetitions\" and comes from \"--reps\" or \"-r\" with default 100.",
-"computetime is \"Computation time (ms)\" and comes from \"--compute\" or \"-c\" with default 129.",
-"",
-"# Allocate a send buffer and a receive buffer.",
-"Task 0 multicasts a numwords*num_tasks word message from buffer 0 to all other tasks.",
-"Task 0 multicasts a numwords*num_tasks word message from buffer 1 to all other tasks.",
-"",
-"# Measure the performance of CODES_MPI_Allreduce().",
-"Task 0 resets its counters then",
-"for reps repetitions {",
-"    all tasks COMPUTES FOR computetime MILLISECONDS then",
-"    all tasks backend execute \"",
-"        CODES_MPI_Allreduce([MESSAGE BUFFER 0], [MESSAGE BUFFER 1], (int)\" and numwords and \",",
-"                MPI_INT, MPI_SUM, MPI_COMM_WORLD);",
-"        \" then",
-"    all tasks backend execute \"",
-"        CODES_MPI_Allreduce([MESSAGE BUFFER 0], [MESSAGE BUFFER 1], (int)\" and numwords and \",",
-"                MPI_INT, MPI_SUM, MPI_COMM_WORLD);",
-"        \"",
-"} then",
-"task 0 logs elapsed_usecs/1000 as \"Elapse time (ms)\".",
-NULL
-};
-
- /* Variables specific to the c_mpi backend */
-int num_tasks;   /* int version of var_num_tasks needed by CODES_MPI_Comm_size() */
-char * procflags;   /* Array of 1s representing an all-task MPI communicator */
-MPI_Comm comm_world = MPI_COMM_WORLD;   /* Copy of MPI_COMM_WORLD that we can take the address of */
-void * attr_val;   /* Pointed to the value of MPI_TAG_UB */
-int attr_flag = 0;   /* true=MPI_TAG_UB was extracted; false=not extracted */
-char log_key_str[128];   /* String representing the range of valid MPI tags */
-
- /* As a special case, if the command line contains --help, then skip
-  * the coNCePTuaL initialization step. */
-for (i=1; i<argc; i++)
-if (!strcmp(argv[i], "--"))
-break;
-else
-if (!strcmp(argv[i], "--help") || !strcmp(argv[i], "-?")) {
-argv[1] = "-?";   /* Guaranteed to work, even with getopt() */
-help_only = 1;
-break;
-}
-
- /* Perform various initializations specific to the c_mpi backend. */
- /* Initialize MPI. */
-//(void) MPI_Init(&argc, &argv);
-mpi_is_running = 1;
-
- /* Initialize the coNCePTuaL run-time library. */
-if (!help_only)
-//ncptl_init (NCPTL_RUN_TIME_VERSION, argv[0]);
-//(void) atexit (conc_exit_handler);
-
- /* Initialize the communication routines needed by the c_mpi backend. */
-//(void) MPI_Errhandler_create ((MPI_Handler_function *)handle_MPI_error, &mpi_error_handler);
-//(void) MPI_Errhandler_set (MPI_COMM_WORLD, mpi_error_handler);
-(void) CODES_MPI_Comm_rank(MPI_COMM_WORLD, &physrank);
-(void) CODES_MPI_Comm_size(MPI_COMM_WORLD, &num_tasks);
-var_num_tasks = (ncptl_int) num_tasks;
-//(void) MPI_Comm_get_attr(MPI_COMM_WORLD, MPI_TAG_UB, &attr_val, &attr_flag);
-mpi_tag_ub = (ncptl_int) (attr_flag ? *(int *)attr_val : 32767);
-
- /* Generate and broadcast a UUID. */
-//logfile_uuid = ncptl_log_generate_uuid();
-//(void) CODES_MPI_Bcast ((void *)logfile_uuid, 37, MPI_CHAR, 0, MPI_COMM_WORLD);
-
- /* Plug variables and default values into the NCPTL_CMDLINE structure. */
-arguments[0].variable = (CMDLINE_VALUE *) &logfiletmpl;
-arguments[1].variable = (CMDLINE_VALUE *) &var_numwords;
-arguments[1].defaultvalue.intval = 29516800LL;
-arguments[2].variable = (CMDLINE_VALUE *) &var_reps;
-arguments[2].defaultvalue.intval = 100LL;
-arguments[3].variable = (CMDLINE_VALUE *) &var_computetime;
-arguments[3].defaultvalue.intval = 129LL;
-//logfiletmpl_default = (char *) ncptl_malloc (strlen(argv0) + 15, 0);
-//sprintf (logfiletmpl_default, "%s-%%p.log", argv0);
-//arguments[0].defaultvalue.stringval = logfiletmpl_default;
-
- /* Parse the command line. */
-mpi_is_running = 0;   /* Don't invoke MPI_Abort() after --help. */
-ncptl_parse_command_line (argc, argv, arguments, sizeof(arguments)/sizeof(NCPTL_CMDLINE));
-if (help_only)
-ncptl_fatal ("Internal error in the c_generic backend: failed to exit after giving help");
-mpi_is_running = 1;
-
- /* Establish a mapping from (virtual) task IDs to (physical) ranks. */
-procmap = ncptl_allocate_task_map (var_num_tasks);
-virtrank = ncptl_physical_to_virtual (procmap, physrank);
-
- /* Perform initializations specific to the c_mpi backend. */
-//ncptl_log_add_comment ("MPI send routines", "MPI_Send() and CODES_MPI_Isend()");
-//ncptl_log_add_comment ("MPI reduction operation", REDUCE_OPERATION_NAME);
-//sprintf (log_key_str, "[0, %" NICS "]", mpi_tag_ub);
-//ncptl_log_add_comment ("MPI tag range", log_key_str);
-//
-// /* Open the log file and write some standard prologue information to it. */
-//logstate = ncptl_log_open (logfiletmpl, physrank);
-////ncptl_log_write_prologue (logstate, argv[0], logfile_uuid, "c_mpi", "C + MPI",
-//var_num_tasks,
-//arguments, sizeof(arguments)/sizeof(NCPTL_CMDLINE),
-//sourcecode);
-//ncptl_free (logfile_uuid);
-
- /* Allocate a variety of dynamically growing queues. */
-eventqueue = ncptl_queue_init (sizeof (CONC_EVENT));
-touchedqueue = ncptl_queue_init (sizeof (ncptl_int));
-
- /* Perform initializations specific to the c_mpi backend. */
-sendreqQ = ncptl_queue_init (sizeof (MPI_Request));
-sendstatQ = ncptl_queue_init (sizeof (MPI_Status));
-recvreqQ = ncptl_queue_init (sizeof (MPI_Request));
-recvstatQ = ncptl_queue_init (sizeof (MPI_Status));
-communicators = ncptl_set_init (ESTIMATED_COMMUNICATORS, var_num_tasks*sizeof(char), sizeof(MPI_Comm));procflags = (char *) ncptl_malloc (var_num_tasks*sizeof(char), 0);
-for (i=0; i<var_num_tasks; i++)
-procflags[i] = 1;
-ncptl_set_insert (communicators, (void *)procflags, (void *)&comm_world);
-ncptl_free (procflags);
-
- /****************************************************
-  * Generated, program-specific initialization code. *
-  ****************************************************/
-
- /* TASK 0LL MULTICAST...TO ALL OTHER TASKS */
-{
-ncptl_int numsenders = 0LL;   /* Number of sending tasks */
-int * sndvol;   /* Number of bytes we send to each other task */
-int * snddisp;   /* Buffer offset of each send */
-int sndnum = 0;   /* Total number of sends from us */
-int * rcvvol;   /* Number of bytes sent to us from each other task */
-int * rcvdisp;   /* Buffer offset of each receive */
-int rcvnum = 0;   /* Total number of sends to us */
-int peervar;   /* Physical rank of one of our peer tasks */
-char * procflags = (char *) ncptl_malloc (var_num_tasks*sizeof(char), 0);   /* Flags indicating whether each task is in or out */
-ncptl_int * sendsfrom = (ncptl_int *) ncptl_malloc (var_num_tasks*sizeof(ncptl_int), 0);   /* Tally of sends from each rank */
-ncptl_int * recvsby = (ncptl_int *) ncptl_malloc (var_num_tasks*sizeof(ncptl_int), 0);   /* Tally of receives by each rank */
-int stasknum;   /* A single source task mapped by ncptl_virtual_to_physical() */
-int ttasknum;   /* A single target task mapped by ncptl_virtual_to_physical() */
-CONC_MCAST_MPI_FUNC mpi_func;   /* The MPI function that will implement the multicast */
-MPI_Comm subcomm;   /* MPI subcommunicator to use */
-
- /* Determine all participants in the many-to-many multicast. */
-memset(procflags, 0, var_num_tasks);
-memset(sendsfrom, 0, var_num_tasks*sizeof(ncptl_int));
-memset(recvsby, 0, var_num_tasks*sizeof(ncptl_int));
-if ((0LL) >= 0 && (0LL) < var_num_tasks) {
-stasknum = (int) ncptl_virtual_to_physical(procmap, 0LL);
-procflags[stasknum] = 1;
-numsenders = 1;
-{
-ncptl_int ivar_a;
-
-for (ivar_a=0; ivar_a<var_num_tasks; ivar_a++)
-if (ivar_a != 0LL) {
-ttasknum = (int) ncptl_virtual_to_physical(procmap, ivar_a);
-sendsfrom[stasknum]++;
-recvsby[ttasknum]++;
-procflags[ttasknum] = 1;
-}
-}
-}
-subcomm = define_MPI_communicator(procflags);
-
- /* Determine if all participants are sending and receiving the same number
-  * and volume of messages.  If so, then we can use the faster CODES_MPI_Alltoall()
-  * function for the multicast instead of the slower CODES_MPI_Alltoallv(). */
-{
-ncptl_int msgtally = -1;   /* Messages sent or received by any participant */
-ncptl_int i;
-mpi_func = CONC_MCAST_MPI_ALLTOALL;  /* Use MPI_Alltoall() unless we require CODES_MPI_Alltoallv(). */
-for (i=0; i<var_num_tasks; i++) {
-stasknum = (int) ncptl_virtual_to_physical(procmap, i);
-if (procflags[stasknum]) {
-if (msgtally == -1)
-msgtally = sendsfrom[stasknum];
-if (sendsfrom[stasknum] != msgtally || recvsby[stasknum] != msgtally) {
-mpi_func = CONC_MCAST_MPI_ALLTOALLV;
-break;
-}
-}
-}
-}
-
- /* The following steps are performed only by those tasks who are
-  * involved in the communication (as senders and/or receivers). */
-if (numsenders > 0 && procflags[physrank]) {
-int groupsize = 0;   /* Number of MPI ranks represented by subcomm */
-ncptl_int ivar_b_loop;
-
-if (numsenders == 1) {
- /* As a special case, use CODES_MPI_Bcast() if there's a single sender. */
-conc_mcast_tallies[CONC_MCAST_MPI_BCAST]++;
-CONC_EVENT *thisev = conc_allocate_event (EV_MCAST);
-thisev->s.mcast.source = ncptl_virtual_to_physical (procmap, 0LL);
-thisev->s.mcast.size = ((var_numwords)*(var_num_tasks))*4LL;
-thisev->s.mcast.alignment = 0LL;
-thisev->s.mcast.misaligned = 0;
-thisev->s.mcast.touching = 0;
-thisev->s.mcast.verification = 0;
-thisev->s.mcast.tag = 0LL;
-thisev->s.mcast.pendingsends = pendingsends;
-thisev->s.mcast.pendingrecvs = pendingrecvs;
-thisev->s.mcast.bufferofs = 0LL;
-if (0LL < 0LL)
-thisev->s.mcast.buffernum = thisev->s.mcast.verification ? pendingsends+pendingrecvs : 0;
-else
-thisev->s.mcast.buffernum = 0LL;
-(void) ncptl_malloc_message (thisev->s.mcast.size+thisev->s.mcast.bufferofs, thisev->s.mcast.alignment, thisev->s.mcast.buffernum, thisev->s.mcast.misaligned);
-thisev->s.mcast.buffer = NULL;
-if (var_num_tasks == 1)
-thisev->s.mcast.source = 0LL;
-thisev->s.mcast.mpi_func = CONC_MCAST_MPI_BCAST;  /* One-to-many communication */
-
-thisev->s.mcast.communicator = subcomm;
-thisev->s.mcast.root = rank_in_MPI_communicator (thisev->s.mcast.communicator, thisev->s.mcast.source);
-if (thisev->s.mcast.tag != 0LL)
-ncptl_fatal ("The c_mpi backend does not support nonzero tags in MULTICAST statements");
-}
-else {
- /* We have more than one sender.
-  * Allocate memory for CODES_MPI_Alltoallv() to use. */
-for (ivar_b_loop=0; ivar_b_loop<var_num_tasks; ivar_b_loop++)
-groupsize += procflags[ivar_b_loop];
-sndvol = (int *) ncptl_malloc(groupsize*sizeof(int), 0);
-memset(sndvol, 0, groupsize*sizeof(int));
-snddisp = (int *) ncptl_malloc(groupsize*sizeof(int), 0);
-memset(snddisp, 0, groupsize*sizeof(int));
-rcvvol = (int *) ncptl_malloc(groupsize*sizeof(int), 0);
-memset(rcvvol, 0, groupsize*sizeof(int));
-rcvdisp = (int *) ncptl_malloc(groupsize*sizeof(int), 0);
-memset(rcvdisp, 0, groupsize*sizeof(int));
-
- /* Determine each task to which I will send data. */
-if ((0LL) == virtrank) {   /* TASK 0LL */
- /* The current coNCePTuaL statement applies to our task. */
-ncptl_int ivar_c_loop;
-for (ivar_c_loop=0; ivar_c_loop<var_num_tasks; ivar_c_loop++)
-if (ivar_c_loop != virtrank) {
- /* In this scope, ivar_c_loop represents a single receiver. */
-peervar = rank_in_MPI_communicator (subcomm, ncptl_virtual_to_physical(procmap, ivar_c_loop));
-sndvol[peervar] = (int) (((var_numwords)*(var_num_tasks))*4LL);
-snddisp[peervar] = (int) (sndnum * (((var_numwords)*(var_num_tasks))*4LL));
-sndnum++;
-}
-}
-
- /* Determine each task who will send data to me. */
-if ((0LL)>=0 && (0LL)<var_num_tasks) {
- /* 0LL now represents one of the tasks that will send to us. */
-ncptl_int ivar_d_task = 0LL;
-if (virtrank != 0LL) {
- /* Register that task ivar_d_task will send us data. */
-peervar = rank_in_MPI_communicator (subcomm, ncptl_virtual_to_physical(procmap, ivar_d_task));
-rcvvol[peervar] = (int) (((var_numwords)*(var_num_tasks))*4LL);
-rcvdisp[peervar] = (int) (rcvnum * (((var_numwords)*(var_num_tasks))*4LL));
-rcvnum++;
-}
-}
-
- /* CODES_MPI_Alltoall() includes sends to self.  Adjust sndnum and rcvnum appropriately. */
-if (mpi_func == CONC_MCAST_MPI_ALLTOALL) {
-CODES_MPI_Comm_size (subcomm, &sndnum);
-rcvnum = sndnum;
-}
-
- /* Prepare to multicast a message. */
-conc_mcast_tallies[mpi_func]++;
-if (1) {
-CONC_EVENT *thisev = conc_allocate_event (EV_MCAST);
-thisev->s.mcast.size = sndnum * (((var_numwords)*(var_num_tasks))*4LL);
-thisev->s.mcast.alignment = 0LL;
-thisev->s.mcast.misaligned = 0;
-thisev->s.mcast.touching = 0;
-thisev->s.mcast.verification = 0;
-thisev->s.mcast.tag = 0LL;
-thisev->s.mcast.pendingsends = pendingsends;
-thisev->s.mcast.pendingrecvs = pendingrecvs;
-thisev->s.mcast.bufferofs = 0LL;
-if (0LL < 0LL)
-thisev->s.mcast.buffernum = thisev->s.mcast.verification ? pendingsends+pendingrecvs : 0;
-else
-thisev->s.mcast.buffernum = 0LL;
-(void) ncptl_malloc_message (thisev->s.mcast.size+thisev->s.mcast.bufferofs, thisev->s.mcast.alignment, thisev->s.mcast.buffernum, thisev->s.mcast.misaligned);
-thisev->s.mcast.buffer = NULL;
-thisev->s.mcast.bufferofs2 = 0LL;
-thisev->s.mcast.size2 = rcvnum * (((var_numwords)*(var_num_tasks))*4LL);
-(void) ncptl_malloc_message (thisev->s.mcast.size2 + thisev->s.mcast.bufferofs2, thisev->s.mcast.alignment, thisev->s.mcast.buffernum+1, thisev->s.mcast.misaligned);
-thisev->s.mcast.buffer2 = NULL;
-thisev->s.mcast.communicator = subcomm;
-if (thisev->s.mcast.tag != 0LL)
-ncptl_fatal ("The c_mpi backend does not support nonzero tags in MULTICAST statements");
-thisev->s.mcast.sndvol = sndvol;
-thisev->s.mcast.snddisp = snddisp;
-thisev->s.mcast.rcvvol = rcvvol;
-thisev->s.mcast.rcvdisp = rcvdisp;
-thisev->s.mcast.source = -1;
-thisev->s.mcast.mpi_func = mpi_func;
-if (mpi_func == CONC_MCAST_MPI_ALLTOALL)
-thisev->s.mcast.sndvol[0] = thisev->s.mcast.rcvvol[0] = ((var_numwords)*(var_num_tasks))*4LL;
-}
-}
-}
-ncptl_free(procflags);
-ncptl_free(sendsfrom);
-ncptl_free(recvsby);
-}
- /* TASK 0LL MULTICAST...TO ALL OTHER TASKS */
-{
-ncptl_int numsenders = 0LL;   /* Number of sending tasks */
-int * sndvol;   /* Number of bytes we send to each other task */
-int * snddisp;   /* Buffer offset of each send */
-int sndnum = 0;   /* Total number of sends from us */
-int * rcvvol;   /* Number of bytes sent to us from each other task */
-int * rcvdisp;   /* Buffer offset of each receive */
-int rcvnum = 0;   /* Total number of sends to us */
-int peervar;   /* Physical rank of one of our peer tasks */
-char * procflags = (char *) ncptl_malloc (var_num_tasks*sizeof(char), 0);   /* Flags indicating whether each task is in or out */
-ncptl_int * sendsfrom = (ncptl_int *) ncptl_malloc (var_num_tasks*sizeof(ncptl_int), 0);   /* Tally of sends from each rank */
-ncptl_int * recvsby = (ncptl_int *) ncptl_malloc (var_num_tasks*sizeof(ncptl_int), 0);   /* Tally of receives by each rank */
-int stasknum;   /* A single source task mapped by ncptl_virtual_to_physical() */
-int ttasknum;   /* A single target task mapped by ncptl_virtual_to_physical() */
-CONC_MCAST_MPI_FUNC mpi_func;   /* The MPI function that will implement the multicast */
-MPI_Comm subcomm;   /* MPI subcommunicator to use */
-
- /* Determine all participants in the many-to-many multicast. */
-memset(procflags, 0, var_num_tasks);
-memset(sendsfrom, 0, var_num_tasks*sizeof(ncptl_int));
-memset(recvsby, 0, var_num_tasks*sizeof(ncptl_int));
-if ((0LL) >= 0 && (0LL) < var_num_tasks) {
-stasknum = (int) ncptl_virtual_to_physical(procmap, 0LL);
-procflags[stasknum] = 1;
-numsenders = 1;
-{
-ncptl_int ivar_e;
-
-for (ivar_e=0; ivar_e<var_num_tasks; ivar_e++)
-if (ivar_e != 0LL) {
-ttasknum = (int) ncptl_virtual_to_physical(procmap, ivar_e);
-sendsfrom[stasknum]++;
-recvsby[ttasknum]++;
-procflags[ttasknum] = 1;
-}
-}
-}
-subcomm = define_MPI_communicator(procflags);
-
- /* Determine if all participants are sending and receiving the same number
-  * and volume of messages.  If so, then we can use the faster CODES_MPI_Alltoall()
-  * function for the multicast instead of the slower CODES_MPI_Alltoallv(). */
-{
-ncptl_int msgtally = -1;   /* Messages sent or received by any participant */
-ncptl_int i;
-mpi_func = CONC_MCAST_MPI_ALLTOALL;  /* Use MPI_Alltoall() unless we require CODES_MPI_Alltoallv(). */
-for (i=0; i<var_num_tasks; i++) {
-stasknum = (int) ncptl_virtual_to_physical(procmap, i);
-if (procflags[stasknum]) {
-if (msgtally == -1)
-msgtally = sendsfrom[stasknum];
-if (sendsfrom[stasknum] != msgtally || recvsby[stasknum] != msgtally) {
-mpi_func = CONC_MCAST_MPI_ALLTOALLV;
-break;
-}
-}
-}
-}
-
- /* The following steps are performed only by those tasks who are
-  * involved in the communication (as senders and/or receivers). */
-if (numsenders > 0 && procflags[physrank]) {
-int groupsize = 0;   /* Number of MPI ranks represented by subcomm */
-ncptl_int ivar_f_loop;
-
-if (numsenders == 1) {
- /* As a special case, use CODES_MPI_Bcast() if there's a single sender. */
-conc_mcast_tallies[CONC_MCAST_MPI_BCAST]++;
-CONC_EVENT *thisev = conc_allocate_event (EV_MCAST);
-thisev->s.mcast.source = ncptl_virtual_to_physical (procmap, 0LL);
-thisev->s.mcast.size = ((var_numwords)*(var_num_tasks))*4LL;
-thisev->s.mcast.alignment = 0LL;
-thisev->s.mcast.misaligned = 0;
-thisev->s.mcast.touching = 0;
-thisev->s.mcast.verification = 0;
-thisev->s.mcast.tag = 0LL;
-thisev->s.mcast.pendingsends = pendingsends;
-thisev->s.mcast.pendingrecvs = pendingrecvs;
-thisev->s.mcast.bufferofs = 0LL;
-if (1LL < 0LL)
-thisev->s.mcast.buffernum = thisev->s.mcast.verification ? pendingsends+pendingrecvs : 0;
-else
-thisev->s.mcast.buffernum = 1LL;
-(void) ncptl_malloc_message (thisev->s.mcast.size+thisev->s.mcast.bufferofs, thisev->s.mcast.alignment, thisev->s.mcast.buffernum, thisev->s.mcast.misaligned);
-thisev->s.mcast.buffer = NULL;
-if (var_num_tasks == 1)
-thisev->s.mcast.source = 0LL;
-thisev->s.mcast.mpi_func = CONC_MCAST_MPI_BCAST;  /* One-to-many communication */
-
-thisev->s.mcast.communicator = subcomm;
-thisev->s.mcast.root = rank_in_MPI_communicator (thisev->s.mcast.communicator, thisev->s.mcast.source);
-if (thisev->s.mcast.tag != 0LL)
-ncptl_fatal ("The c_mpi backend does not support nonzero tags in MULTICAST statements");
-}
-else {
- /* We have more than one sender.
-  * Allocate memory for CODES_MPI_Alltoallv() to use. */
-for (ivar_f_loop=0; ivar_f_loop<var_num_tasks; ivar_f_loop++)
-groupsize += procflags[ivar_f_loop];
-sndvol = (int *) ncptl_malloc(groupsize*sizeof(int), 0);
-memset(sndvol, 0, groupsize*sizeof(int));
-snddisp = (int *) ncptl_malloc(groupsize*sizeof(int), 0);
-memset(snddisp, 0, groupsize*sizeof(int));
-rcvvol = (int *) ncptl_malloc(groupsize*sizeof(int), 0);
-memset(rcvvol, 0, groupsize*sizeof(int));
-rcvdisp = (int *) ncptl_malloc(groupsize*sizeof(int), 0);
-memset(rcvdisp, 0, groupsize*sizeof(int));
-
- /* Determine each task to which I will send data. */
-if ((0LL) == virtrank) {   /* TASK 0LL */
- /* The current coNCePTuaL statement applies to our task. */
-ncptl_int ivar_g_loop;
-for (ivar_g_loop=0; ivar_g_loop<var_num_tasks; ivar_g_loop++)
-if (ivar_g_loop != virtrank) {
- /* In this scope, ivar_g_loop represents a single receiver. */
-peervar = rank_in_MPI_communicator (subcomm, ncptl_virtual_to_physical(procmap, ivar_g_loop));
-sndvol[peervar] = (int) (((var_numwords)*(var_num_tasks))*4LL);
-snddisp[peervar] = (int) (sndnum * (((var_numwords)*(var_num_tasks))*4LL));
-sndnum++;
-}
-}
-
- /* Determine each task who will send data to me. */
-if ((0LL)>=0 && (0LL)<var_num_tasks) {
- /* 0LL now represents one of the tasks that will send to us. */
-ncptl_int ivar_h_task = 0LL;
-if (virtrank != 0LL) {
- /* Register that task ivar_h_task will send us data. */
-peervar = rank_in_MPI_communicator (subcomm, ncptl_virtual_to_physical(procmap, ivar_h_task));
-rcvvol[peervar] = (int) (((var_numwords)*(var_num_tasks))*4LL);
-rcvdisp[peervar] = (int) (rcvnum * (((var_numwords)*(var_num_tasks))*4LL));
-rcvnum++;
-}
-}
-
- /* CODES_MPI_Alltoall() includes sends to self.  Adjust sndnum and rcvnum appropriately. */
-if (mpi_func == CONC_MCAST_MPI_ALLTOALL) {
-CODES_MPI_Comm_size (subcomm, &sndnum);
-rcvnum = sndnum;
-}
-
- /* Prepare to multicast a message. */
-conc_mcast_tallies[mpi_func]++;
-if (1) {
-CONC_EVENT *thisev = conc_allocate_event (EV_MCAST);
-thisev->s.mcast.size = sndnum * (((var_numwords)*(var_num_tasks))*4LL);
-thisev->s.mcast.alignment = 0LL;
-thisev->s.mcast.misaligned = 0;
-thisev->s.mcast.touching = 0;
-thisev->s.mcast.verification = 0;
-thisev->s.mcast.tag = 0LL;
-thisev->s.mcast.pendingsends = pendingsends;
-thisev->s.mcast.pendingrecvs = pendingrecvs;
-thisev->s.mcast.bufferofs = 0LL;
-if (1LL < 0LL)
-thisev->s.mcast.buffernum = thisev->s.mcast.verification ? pendingsends+pendingrecvs : 0;
-else
-thisev->s.mcast.buffernum = 1LL;
-(void) ncptl_malloc_message (thisev->s.mcast.size+thisev->s.mcast.bufferofs, thisev->s.mcast.alignment, thisev->s.mcast.buffernum, thisev->s.mcast.misaligned);
-thisev->s.mcast.buffer = NULL;
-thisev->s.mcast.bufferofs2 = 0LL;
-thisev->s.mcast.size2 = rcvnum * (((var_numwords)*(var_num_tasks))*4LL);
-(void) ncptl_malloc_message (thisev->s.mcast.size2 + thisev->s.mcast.bufferofs2, thisev->s.mcast.alignment, thisev->s.mcast.buffernum+1, thisev->s.mcast.misaligned);
-thisev->s.mcast.buffer2 = NULL;
-thisev->s.mcast.communicator = subcomm;
-if (thisev->s.mcast.tag != 0LL)
-ncptl_fatal ("The c_mpi backend does not support nonzero tags in MULTICAST statements");
-thisev->s.mcast.sndvol = sndvol;
-thisev->s.mcast.snddisp = snddisp;
-thisev->s.mcast.rcvvol = rcvvol;
-thisev->s.mcast.rcvdisp = rcvdisp;
-thisev->s.mcast.source = -1;
-thisev->s.mcast.mpi_func = mpi_func;
-if (mpi_func == CONC_MCAST_MPI_ALLTOALL)
-thisev->s.mcast.sndvol[0] = thisev->s.mcast.rcvvol[0] = ((var_numwords)*(var_num_tasks))*4LL;
-}
-}
-}
-ncptl_free(procflags);
-ncptl_free(sendsfrom);
-ncptl_free(recvsby);
-}
- /* TASK 0LL RESET THEIR COUNTERS */
-if ((0LL) == virtrank) {   /* TASK 0LL */
- /* The current coNCePTuaL statement applies to our task. */
-(void) conc_allocate_event (EV_RESET);
-}
- /* THEN... */
- /* FOR var_reps REPETITIONS... */
-{
-ncptl_int numreps = var_reps;   /* Total # of repetitions to perform */
-int unroll_loop = numreps <= CONC_MAX_UNROLL;   /* 1=unroll loop; 0=use a REPEAT event */
-ncptl_int repnum;   /* Current repetition number */
-CONC_EVENT * repevent = NULL;   /* Event designating repetition */
-ncptl_int repeventnum = -1LL;   /* Event number corresponding to repevent */
- /* Conditionally unroll the loop. */
-for (repnum=0; repnum<(unroll_loop?numreps:1); repnum++) {
- /* Allocate a repeat event if we have more than one repetition. */
-if (!unroll_loop && numreps > 1LL) {
-repeventnum = ncptl_queue_length (eventqueue);
-repevent =  conc_allocate_event (EV_REPEAT);
-repevent->s.rep.numreps = numreps;
-}
-
- /* Output a loop body if we have at least one repetition. */
-if (unroll_loop || numreps > 0LL) {
- /* ALL TASKS COMPUTES FOR (var_computetime)*(1000LL) MICROSECONDS */
-{   /* ALL TASKS */
- /* The current coNCePTuaL statement applies to our task. */
-CONC_EVENT *thisev = conc_allocate_event (EV_DELAY);
-thisev->s.delay.microseconds = (uint64_t) var_computetime;
-thisev->s.delay.microseconds *= (uint64_t) 1000LL;
-thisev->s.delay.spin0sleep1 = 0;
-}
- /* THEN... */
- /* ALL TASKS BACKEND EXECUTES ['"\\n        CODES_MPI_Allreduce([MESSAGE BUFFER 0], [MESSAGE BUFFER 1], (int)"', 'var_numwords', '",\\n                MPI_INT, MPI_SUM, MPI_COMM_WORLD);\\n        "'] */
-{   /* ALL TASKS */
- /* The current coNCePTuaL statement applies to our task. */
-CONC_EVENT *thisev = conc_allocate_event (EV_CODE);
-thisev->s.code.number = 0;
-thisev->s.code.procmap = NULL;
-thisev->s.code.var_numwords = var_numwords;
-}
- /* THEN... */
- /* ALL TASKS BACKEND EXECUTES ['"\\n        CODES_MPI_Allreduce([MESSAGE BUFFER 0], [MESSAGE BUFFER 1], (int)"', 'var_numwords', '",\\n                MPI_INT, MPI_SUM, MPI_COMM_WORLD);\\n        "'] */
-{   /* ALL TASKS */
- /* The current coNCePTuaL statement applies to our task. */
-CONC_EVENT *thisev = conc_allocate_event (EV_CODE);
-thisev->s.code.number = 1;
-thisev->s.code.procmap = NULL;
-thisev->s.code.var_numwords = var_numwords;
-}
-}
-}
-
- /* Assign the number of events to repeat, now that we know that number. */
-if (!unroll_loop && numreps > 1LL) {
-repevent = repeventnum + (CONC_EVENT *) ncptl_queue_contents (eventqueue, 0);
-repevent->s.rep.end_event = ncptl_queue_length (eventqueue) - 1;
-}
-}
- /* THEN... */
- /* TASK 0LL LOGS "Elapse time (ms)" */
-if ((0LL) == virtrank) {   /* TASK 0LL */
- /* The current coNCePTuaL statement applies to our task. */
-CONC_EVENT *thisev = conc_allocate_event (EV_CODE);
-thisev->s.code.number = 2;
-thisev->s.code.procmap = NULL;
-}
- /* Begin a new top-level statement. */
-(void) conc_allocate_event (EV_NEWSTMT);
-
- /*************************
-  * More boilerplate code *
-  *************************/
-
- /* Abort if the program will terminate with pending messages. */
-if (pendingsends && pendingrecvs)
-ncptl_fatal("Neglected to await the completion of %" NICS " asynchronous %s and %" NICS " asynchronous %s",
-pendingsends, pendingsends==1LL ? "send" : "sends",
-pendingrecvs, pendingrecvs==1LL ? "receive" : "receives");
-else
-if (pendingsends)
-ncptl_fatal("Neglected to await the completion of %" NICS " asynchronous %s",
-pendingsends, pendingsends==1LL ? "send" : "sends");
-else
-if (pendingrecvs)
-ncptl_fatal("Neglected to await the completion of %" NICS " asynchronous %s",
-pendingrecvs, pendingrecvs==1LL ? "receive" : "receives");
-
- /* Allocate memory for non-unique messages and asynchronous
-  * message handles now that we know how much memory we need
-  * to allocate. */
-eventlist = (CONC_EVENT *) ncptl_queue_contents (eventqueue, 0);
-numevents = ncptl_queue_length (eventqueue);
-sendrequests = (MPI_Request *) ncptl_queue_contents (sendreqQ, 0);
-recvrequests = (MPI_Request *) ncptl_queue_contents (recvreqQ, 0);
-for (i=0; i<numevents; i++) {
-CONC_EVENT *thisev = &eventlist[i];   /* Cache of the current event */
-switch (thisev->type) {
-case EV_MCAST:
-if (!thisev->s.mcast.buffer)
-thisev->s.mcast.buffer = ncptl_malloc_message (thisev->s.mcast.bufferofs + thisev->s.mcast.size,
-thisev->s.mcast.alignment,
-thisev->s.mcast.buffernum,
-thisev->s.mcast.misaligned);
-if (thisev->s.mcast.verification)
-ncptl_fill_buffer (CONC_GETBUFPTR(mcast), thisev->s.mcast.bufferofs + thisev->s.mcast.size, -1);
-if (thisev->s.mcast.mpi_func == CONC_MCAST_MPI_ALLTOALL || thisev->s.mcast.mpi_func == CONC_MCAST_MPI_ALLTOALLV)
-thisev->s.mcast.buffer2 = ncptl_malloc_message (thisev->s.mcast.bufferofs2 + thisev->s.mcast.size2,
-thisev->s.mcast.alignment,
-thisev->s.mcast.buffernum + 1,
-thisev->s.mcast.misaligned);
-break;
-
-default:
-break;
-}
-}
-}
-
-/* Process a subset of the events in a given event list. */
-static void conc_process_events (CONC_EVENT *eventlist,
-ncptl_int firstev, ncptl_int lastev, ncptl_int numreps)
-{
-CONC_EVENT * thisev;   /* Cache of the current event */
-CONC_EVENT * thisev_first = &eventlist[firstev];   /* Cache of the first event */
-ncptl_int i;   /* Iterate over events. */
-ncptl_int j;   /* Iterate over repetitions. */
-
- /* Process from event firstev to event lastev (both inclusive). */
-for (j=numreps; j>0; j--)
-for (i=firstev, thisev=thisev_first; i<=lastev; i++, thisev++) {
- /* Declare variables needed by all C-based backends. */
-
- /* Process a single event. */
-switch (thisev->type) {
-case EV_MCAST:
- /* Synchronous multicast */
-switch (thisev->s.mcast.mpi_func) {
-case CONC_MCAST_MPI_BCAST:
- /* One to many */
-(void) CODES_MPI_Bcast (CONC_GETBUFPTR(mcast), thisev->s.mcast.size, MPI_BYTE,
-thisev->s.mcast.root, thisev->s.mcast.communicator);
-break;
-case CONC_MCAST_MPI_ALLTOALL:
- /* Many to many, same to each */
-(void) CODES_MPI_Alltoall (CONC_GETBUFPTR(mcast), thisev->s.mcast.sndvol[0], MPI_BYTE,
-(void *)((char *)thisev->s.mcast.buffer2 + thisev->s.mcast.bufferofs2), thisev->s.mcast.rcvvol[0], MPI_BYTE, thisev->s.mcast.communicator);
-break;
-case CONC_MCAST_MPI_ALLTOALLV:
- /* Many to many, different to each */
-(void) CODES_MPI_Alltoallv (CONC_GETBUFPTR(mcast), thisev->s.mcast.sndvol, thisev->s.mcast.snddisp, MPI_BYTE,
-(void *)((char *)thisev->s.mcast.buffer2 + thisev->s.mcast.bufferofs2), thisev->s.mcast.rcvvol, thisev->s.mcast.rcvdisp, MPI_BYTE, thisev->s.mcast.communicator);
-break;
-default:
- /* We should never get here. */
-ncptl_fatal ("Internal error: Unrecognized multicast function");
-break;
-}
-break;
-
-case EV_DELAY:
- /* Do nothing for a prescribed length of time. */
-ncptl_udelay (thisev->s.delay.microseconds, thisev->s.delay.spin0sleep1);
-break;
-
-case EV_RESET:
- /* Reset all of the counters exported to coNCePTuaL programs. */
-starttime = ncptl_time();
-break;
-
-case EV_REPEAT:
- /* Repeatedly perform the next batch of events. */
-conc_process_events (eventlist, i+1, thisev->s.rep.end_event, thisev->s.rep.numreps);
-i = thisev->s.rep.end_event;
-thisev = &eventlist[i];
-break;
-
-case EV_NEWSTMT:
- /* Begin a new table in the log file. */
-if (!suppress_output) {
-uint64_t stop_elapsed_usecs = ncptl_time();
-var_elapsed_usecs = stop_elapsed_usecs - starttime;
-//ncptl_log_commit_data (logstate);
-starttime += ncptl_time() - stop_elapsed_usecs;
-}
-break;
-
-case EV_CODE:
- /* Execute an arbitrary piece of code. */
-switch (thisev->s.code.number) {
-case 0:
- /* ALL TASKS BACKEND EXECUTES ['"\\n        CODES_MPI_Allreduce([MESSAGE BUFFER 0], [MESSAGE BUFFER 1], (int)"', 'var_numwords', '",\\n                MPI_INT, MPI_SUM, MPI_COMM_WORLD);\\n        "'] */
-{
-
-        CODES_MPI_Allreduce((ncptl_get_message_buffer ((ncptl_int)(0))), (ncptl_get_message_buffer ((ncptl_int)(1))), (int) (double)thisev->s.code.var_numwords ,
-                MPI_INT, MPI_SUM, MPI_COMM_WORLD);
-        
-}
-break;
-
-case 1:
- /* ALL TASKS BACKEND EXECUTES ['"\\n        CODES_MPI_Allreduce([MESSAGE BUFFER 0], [MESSAGE BUFFER 1], (int)"', 'var_numwords', '",\\n                MPI_INT, MPI_SUM, MPI_COMM_WORLD);\\n        "'] */
-{
-
-        CODES_MPI_Allreduce((ncptl_get_message_buffer ((ncptl_int)(0))), (ncptl_get_message_buffer ((ncptl_int)(1))), (int) (double)thisev->s.code.var_numwords ,
-                MPI_INT, MPI_SUM, MPI_COMM_WORLD);
-        
-}
-break;
-
-case 2:
- /* TASK 0LL LOGS "Elapse time (ms)" */
-if (!suppress_output) {
-uint64_t stop_elapsed_usecs = ncptl_time();
-var_elapsed_usecs = stop_elapsed_usecs - starttime;
-//ncptl_log_write (logstate, 0, "Elapse time (ms)", NCPTL_FUNC_NO_AGGREGATE, 0.0, ((double)var_elapsed_usecs)/(1000.0));
-starttime += ncptl_time() - stop_elapsed_usecs;
-}
-break;
-
-default:
- /* The C code generation module must be broken. */
-ncptl_fatal ("Internal error: unknown EV_CODE block %" NICS, thisev->s.code.number);
-break;
-}
-break;
-
-default:
- /* The c_generic backend or the c_mpi backend must be broken. */
-ncptl_fatal ("Internal error: unknown event type %d", thisev->type);
-break;
-}
-}
-}
-
-/* Finish up cleanly and return a status code. */
-static int conc_finalize (void)
-{
-int exitcode = 0;   /* Program exit code (to pass to exit()) */
-
- /* Declarations specific to the c_mpi backend */
-//int mpiresult;   /* Return code from MPI_Finalize() */
-//char log_key_str[128];   /* String representing the range of valid MPI tags */
-//
-// /* Finalization code specific to the c_mpi backend */
-//log_key_str[0] = '\0';
-//if (conc_mcast_tallies[CONC_MCAST_MPI_BCAST] > 0) {
-//char onefuncstr[50];
-//sprintf (onefuncstr, "%sCODES_MPI_Bcast()*%" NICS,
-//log_key_str[0] == '\0' ? "" : " ", conc_mcast_tallies[CONC_MCAST_MPI_BCAST]);
-//strcat (log_key_str, onefuncstr);
-//}
-//if (conc_mcast_tallies[CONC_MCAST_MPI_ALLTOALL] > 0) {
-//char onefuncstr[50];
-//sprintf (onefuncstr, "%sCODES_MPI_Alltoall()*%" NICS,
-//log_key_str[0] == '\0' ? "" : " ", conc_mcast_tallies[CONC_MCAST_MPI_ALLTOALL]);
-//strcat (log_key_str, onefuncstr);
-//}
-//if (conc_mcast_tallies[CONC_MCAST_MPI_ALLTOALLV] > 0) {
-//char onefuncstr[50];
-//sprintf (onefuncstr, "%sCODES_MPI_Alltoallv()*%" NICS,
-//log_key_str[0] == '\0' ? "" : " ", conc_mcast_tallies[CONC_MCAST_MPI_ALLTOALLV]);
-//strcat (log_key_str, onefuncstr);
-//}
-//if (log_key_str[0] != '\0')
-//ncptl_log_add_comment ("Multicast functions used (statically)", log_key_str);
-//
-// /* Write a standard epilogue to the log file. */
-////ncptl_log_commit_data (logstate);
-////ncptl_log_write_epilogue (logstate);
-//ncptl_log_close (logstate);
-
- /* Inform the run-time library that it's no longer needed. */
-//ncptl_queue_empty (eventqueue);
-//ncptl_free (eventqueue);
-//ncptl_finalize();
-
- /* Finalization code specific to the c_mpi backend */
-CODES_MPI_Finalize();
-mpi_is_running = 0;
-exitcode = 0;
-
- /* Return an exit status code. */
-return exitcode;
-}
-
-/*************************************************************************/
-/***************************** MAIN ROUTINE ******************************/
-/*************************************************************************/
-
-/* Program execution starts here. */
-static int cosmoflow_main (int argc, char *argv[])
-{
- /* Declare variables needed by all C-based backends. */
-CONC_EVENT * eventlist;   /* List of events to execute */
-ncptl_int numevents;   /* Number of entries in eventlist[] */
-
- /* ----- Initialization ----- */
-conc_initialize (argc, argv);
-eventlist = (CONC_EVENT *) ncptl_queue_contents (eventqueue, 0);
-numevents = ncptl_queue_length (eventqueue);
-sendrequests = (MPI_Request *) ncptl_queue_contents (sendreqQ, 0);
-sendstatuses = (MPI_Status *) ncptl_queue_contents (sendstatQ, 0);
-recvrequests = (MPI_Request *) ncptl_queue_contents (recvreqQ, 0);
-recvstatuses = (MPI_Status *) ncptl_queue_contents (recvstatQ, 0);
-starttime = ncptl_time();
-
- /* ----- Event-list processing ----- */
-conc_process_events (eventlist, 0, numevents-1, 1);
-
- /* ----- Finalization ----- */
-return conc_finalize();
-}
-
-/* fill in function pointers for this method */
-struct codes_conceptual_bench cosmoflow_bench = 
-{
-.program_name = "cosmoflow",
-.conceptual_main = cosmoflow_main,
-};

From c1b294d44a0fa523c8960efe3c18dedc157fcddb Mon Sep 17 00:00:00 2001
From: Xin Wang <xwang149@hawk.iit.edu>
Date: Tue, 22 Jan 2019 15:14:33 -0600
Subject: [PATCH 12/15] Delete conc-hotpotato.c

---
 .../conceputal-skeleton-apps/conc-hotpotato.c | 1266 -----------------
 1 file changed, 1266 deletions(-)
 delete mode 100644 src/workload/conceputal-skeleton-apps/conc-hotpotato.c

diff --git a/src/workload/conceputal-skeleton-apps/conc-hotpotato.c b/src/workload/conceputal-skeleton-apps/conc-hotpotato.c
deleted file mode 100644
index 27f0da78..00000000
--- a/src/workload/conceputal-skeleton-apps/conc-hotpotato.c
+++ /dev/null
@@ -1,1266 +0,0 @@
-/**********************************************************************
- * This file was generated by coNCePTuaL on Mon Jan  7 23:10:01 2019
- * using the c_mpi backend (C + MPI).
- * Do not modify this file; modify /Users/xin/macworkspace/codes-dev/codes/scripts/conceptual_benchmarks/hotpotato.ncptl instead.
- *
- * Entire source program
- * ---------------------
- *   # Virtual ring "hot potato" test
- *   
- *   Require language version "1.5".
- *   
- *   trials is "Number of trials to perform" and comes from "--trials" or
- *     "-t" with default 100000.
- *   
- *   Assert that "the hot-potato test requires at least two tasks" with num_tasks>=2.
- *   
- *   Let len be 0 while {
- *     for each task_count in {2, ..., num_tasks} {
- *       task 0 outputs "Performing " and trials and " " and
- *   		   task_count and "-task runs...." then
- *       for trials repetitions plus 5 warmup repetitions and a synchronization {
- *         task 0 resets its counters then
- *         task 0 sends a len byte message to unsuspecting task 1 then
- *         task (n+1) mod task_count receives a len byte message from task n such that n<task_count then
- *         task n such that n>0 /\ n<task_count sends a len byte message to unsuspecting task (n+1) mod task_count then
- *         task 0 logs the task_count as "# of tasks" and
- *   		  the minimum of elapsed_usecs/task_count as "Latency (usecs)" and
- *   		  the mean of elapsed_usecs/task_count as "Latency (usecs)" and
- *   		  the variance of elapsed_usecs/task_count as "Latency (usecs)"
- *       } then
- *       task 0 computes aggregates
- *     }
- *   }
- **********************************************************************/
-
-#include "codes/codes-conc-addon.h"
-/*****************
- * Include files *
- *****************/
-
-/* Header files needed by all C-based backends */
-#include <stdio.h>
-#include <string.h>
-#include <ncptl/ncptl.h>
-
-/* Header files specific to the c_mpi backend */
-#include <mpi.h>
-#include <stdarg.h>
-
-/**********
- * Macros *
- **********/
-
-/* Define the maximum loop trip count that we're willing to unroll fully. */
-#define CONC_MAX_UNROLL 5
-
-/* Specify the minimum number of trial iterations in each FOR <time> loop. */
-#define CONC_FOR_TIME_TRIALS 1
-
-/* Define a macro that rounds a double to a ncptl_int. */
-#define CONC_DBL2INT(D) ((ncptl_int)((D)+0.5))
-
-/* Define a macro that increments a buffer pointer by a byte offset. */
-#define CONC_GETBUFPTR(S) ((void *)((char *)thisev->s.S.buffer + thisev->s.S.bufferofs))
-
-/* Implement ncptl_func_task_of in terms of ncptl_physical_to_virtual. */
-#define ncptl_func_task_of(P) ((ncptl_int)(P) < 0LL || (ncptl_int)(P) >= var_num_tasks ? -1LL : ncptl_physical_to_virtual (procmap, (ncptl_int)(P)))
-#define ncptl_dfunc_task_of(P) ((double) ncptl_func_task_of(P))
-
-/* Implement ncptl_func_processor_of in terms of ncptl_virtual_to_physical. */
-#define ncptl_func_processor_of(V) ((ncptl_int)(V) < 0LL || (ncptl_int)(V) >= var_num_tasks ? -1LL : ncptl_virtual_to_physical (procmap, (ncptl_int)(V)))
-#define ncptl_dfunc_processor_of(V) ((double) ncptl_func_processor_of(V))
-
-/* Define a macro that increments REDUCE's alternate buffer pointer by a byte offset. */
-#define CONC_GETALTBUFPTR(S) ((void *)((char *)thisev->s.S.altbuffer + thisev->s.S.bufferofs))
-
-/* Estimate the number of unique communicators that this program will need.
- * (The tradeoff is one of initialization time versus memory consumption.) */
-#define ESTIMATED_COMMUNICATORS 128
-
-/* Specify an operation to use for all reduction operations. */
-#define REDUCE_OPERATION MPI_SUM
-#define REDUCE_OPERATION_NAME "MPI_SUM"
-
-
-/*********************
- * Type declarations *
- *********************/
-
-/* Enumerate the various mechanisms used to implement MULTICAST statements. */
-typedef enum {
-CONC_MCAST_MPI_BCAST,       /* One to many */
-CONC_MCAST_MPI_ALLTOALL,    /* Many to many, same data to all */ 
-CONC_MCAST_MPI_ALLTOALLV,   /* General many to many */
-CONC_MCAST_MPI_NUM_FUNCS    /* Number of the above */
-} CONC_MCAST_MPI_FUNC;
-
-/* Define the type of event to perform. */
-typedef enum {
-EV_SEND,     /* Synchronous send */
-EV_ASEND,    /* Asynchronous send */
-EV_RECV,     /* Synchronous receive */
-EV_ARECV,    /* Asynchronous receive */
-EV_WAIT,     /* Wait for all asynchronous sends/receives to complete */
-EV_DELAY,    /* Spin or sleep */
-EV_TOUCH,    /* Touch a region of memory */
-EV_SYNC,     /* Barrier synchronization */
-EV_RESET,    /* Reset counters */
-EV_STORE,    /* Store all counters' current values */
-EV_RESTORE,  /* Restore the previously pushed counter values */
-EV_FLUSH,    /* Compute aggregate functions for log-file columns */
-EV_MCAST,    /* Synchronous multicast */
-EV_REDUCE,   /* Reduction with or without a subsequent multicast */
-EV_BTIME,    /* Beginning of a timed loop */
-EV_ETIME,    /* Ending of a timed loop */
-EV_REPEAT,   /* Repeatedly process the next N events */
-EV_SUPPRESS, /* Suppress writing to the log and standard output */
-EV_NEWSTMT,  /* Beginning of a new top-level statement */
-EV_CODE,     /* None of the above */
-NUM_EVS      /* Number of event types in CONC_EVENT_TYPE */
-} CONC_EVENT_TYPE;
-
-/* Describe a synchronous or asynchronous send event. */
-typedef struct {
-ncptl_int dest;         /* Destination task */
-ncptl_int size;         /* Number of bytes to send */
-ncptl_int alignment;    /* Message alignment (in bytes) */
-ncptl_int pendingsends; /* # of outstanding sends */
-ncptl_int pendingrecvs; /* # of outstanding receives */
-ncptl_int buffernum;    /* Buffer # to send from */
-ncptl_int bufferofs;    /* Byte offset into the message buffer */
-ncptl_int tag;          /* Tag to use for selective receives */
-int misaligned;         /* 1=misaligned from a page; 0=align as specified */
-int touching;           /* 1=touch every word before sending */
-int verification;       /* 1=fill message buffer with known contents */
-void *buffer;           /* Pointer to message memory */
-MPI_Request * handle;   /* MPI handle representing an asynchronous send */
-} CONC_SEND_EVENT;
-
-/* Describe a synchronous or asynchronous receive event. */
-typedef struct {
-ncptl_int source;       /* Source task */
-ncptl_int size;         /* Number of bytes to receive */
-ncptl_int alignment;    /* Message alignment (in bytes) */
-ncptl_int pendingsends; /* # of outstanding sends */
-ncptl_int pendingrecvs; /* # of outstanding receives */
-ncptl_int buffernum;    /* Buffer # to receive into */
-ncptl_int bufferofs;    /* Byte offset into the message buffer */
-ncptl_int tag;          /* Tag to use for selective receives */
-int misaligned;         /* 1=misaligned from a page; 0=align as specified */
-int touching;           /* 1=touch every word after reception */
-int verification;       /* 1=verify that all bits are correct */
-void *buffer;           /* Pointer to message memory */
-MPI_Request * handle;   /* MPI handle representing an asynchronous receive */
-} CONC_RECV_EVENT;
-
-/* Describe a wait-for-asynchronous-completions event. */
-typedef struct {
-ncptl_int numsends;     /* # of sends we expect to complete. */
-ncptl_int numrecvs;     /* # of receives we expect to complete. */
-ncptl_int numrecvbytes; /* # of bytes we expect to receive-complete */
-ncptl_int *touchedlist;    /* List of receives that need to be touched */
-ncptl_int numtouches;         /* # of elements in the above */
-} CONC_WAIT_EVENT;
-
-/* Describe a spin or sleep delay. */
-typedef struct {
-uint64_t microseconds;  /* Length of delay in microseconds */
-int spin0sleep1;        /* 0=spin; 1=sleep */
-} CONC_DELAY_EVENT;
-
-/* Describe a barrier synchronization event. */
-typedef struct {
-MPI_Comm communicator;   /* Set of tasks to synchronize */
-} CONC_SYNC_EVENT;
-
-/* Describe a walk over a memory-region. */
-typedef struct {
-ncptl_int regionbytes;  /* Size in bytes of the region to touch */
-ncptl_int bytestride;   /* Stride in bytes to touch */
-ncptl_int numaccesses;  /* Number of words to touch */
-ncptl_int wordsize;     /* Size in bytes of each touch */
-ncptl_int firstbyte;    /* Byte offset of the first byte to touch */
-} CONC_TOUCH_EVENT;
-
-/* Describe a synchronous multicast event. */
-typedef struct {
-ncptl_int source;       /* Source task, -1 in the many-to-many case */
-ncptl_int size;         /* Number of bytes to send */
-ncptl_int alignment;    /* Message alignment (in bytes) */
-ncptl_int pendingsends; /* # of outstanding sends */
-ncptl_int pendingrecvs; /* # of outstanding receives */
-ncptl_int buffernum;    /* Buffer # to send/receive from */
-ncptl_int bufferofs;    /* Byte offset into the message buffer */
-ncptl_int tag;          /* Tag to use for selective receives */
-int misaligned;         /* 1=misaligned from a page; 0=align as specified */
-int touching;           /* 1=touch every word before sending */
-int verification;       /* 1=verify that all bits are correct */
-void *buffer;           /* Pointer to message memory */
-ncptl_int size2;   /* Number of bytes to receive in the many-to-many case */
-ncptl_int bufferofs2;   /* Byte offset into the message buffer in the many-to-many case */
-void * buffer2;   /* Pointer to receive-message memory in the many-to-many case */
-MPI_Comm communicator;   /* Set of tasks to multicast to/from */
-int root;   /* source's rank within communicator */
-int * sndvol;   /* Volume of data to send to each rank in the communicator */
-int * snddisp;   /* Offset from buffer of each message to send */
-int * rcvvol;   /* Volume of data to receive from each rank in the communicator */
-int * rcvdisp;   /* Offset from buffer2 of each message to receive */
-CONC_MCAST_MPI_FUNC mpi_func;   /* MPI function to use to perform the multicast */
-} CONC_MCAST_EVENT;
-
-/* Describe a reduction event. */
-typedef struct {
-ncptl_int numitems;     /* # of items to reduce */
-ncptl_int itemsize;     /* # of bytes per item */
-ncptl_int alignment;    /* Message alignment (in bytes) */
-ncptl_int pendingsends; /* # of outstanding sends */
-ncptl_int pendingrecvs; /* # of outstanding receives */
-ncptl_int buffernum;    /* Buffer # to send/receive from */
-ncptl_int bufferofs;    /* Byte offset into the message buffer */
-ncptl_int tag;          /* Tag to use for selective receives */
-int misaligned;         /* 1=misaligned from a page; 0=align as specified */
-int touching;           /* 1=touch every word before sending/after receiving */
-int sending;            /* 1=we're a sender */
-int receiving;          /* 1=we're a receiver */
-void *buffer;           /* Pointer to message memory */
-void * altbuffer;   /* Pointer to additional message memory */
-MPI_Comm sendcomm;   /* Set of tasks to reduce from */
-MPI_Comm recvcomm;   /* Set of tasks to reduce to */
-MPI_Datatype datatype;   /* MPI datatype to reduce */
-int reducetype;   /* 0=reduce; 1=allreduce; 2=reduce+bcast */
-ncptl_int reduceroot;   /* Root task of the reduction if reducetype is 0 or 2 */
-ncptl_int bcastroot;   /* Root task of the multicast if reducetype is 2 */
-} CONC_REDUCE_EVENT;
-
-/* Describe an event representing the beginning of a timed loop. */
-typedef struct {
-uint64_t usecs;         /* Requested loop duration */
-uint64_t warmup_usecs;  /* Requested duration of warmup loops */
-uint64_t starttime;     /* Time at which the loop state last changed */
-uint64_t itersleft;     /* # of iterations remaining */
-uint64_t previters;     /* # of iterations we performed last time */
-int prev_quiet;         /* Previous value of suppress_output */
-int timing_trial;       /* 1=performing a timing trial; 0=running for real */
-volatile int finished;  /* 1=time has expired; 0=still ticking */
-} CONC_BTIME_EVENT;
-
-/* Describe an event representing the end of a timed loop. */
-typedef struct {
-ncptl_int begin_event;  /* Index into eventlist[] of the corresponding BTIME event */
-} CONC_ETIME_EVENT;
-
-/* Describe an event representing repetitions of subsequent events. */
-typedef struct {
-ncptl_int end_event;    /* Index into eventlist[] of the last event to repeat */
-ncptl_int numreps;      /* # of repetitions to perform */
-} CONC_REPEAT_EVENT;
-
-/* Describe an event representing output suppression (either on or off). */
-typedef struct conc_suppress_event {
-int quiet;              /* 0=allow output; 1=suppress it */
-int prev_quiet;         /* Previous value of suppress_output */
-ncptl_int matching_event;  /* Event ID of the "suppression on" event */
-uint64_t stop_elapsed_usecs;   /* Time at which we suppressed output */
-} CONC_SUPPRESS_EVENT;
-
-/* Describe an event representing arbitrary code to execute at run time. */
-typedef struct {
-ncptl_int number;       /* Unique number corresponding to a specific piece of code */NCPTL_VIRT_PHYS_MAP *procmap;  /* Current mapping between tasks and processors */
-ncptl_int var_task_count;   /* Copy of var_task_count to use within a piece of code */
-ncptl_int var_trials;   /* Copy of var_trials to use within a piece of code */
-} CONC_CODE_EVENT;
-
-/* Describe an arbitrary coNCePTuaL event. */
-typedef struct {
-CONC_EVENT_TYPE type;          /* Type of event */
-union {
-CONC_SEND_EVENT send;          /* Send state */
-CONC_RECV_EVENT recv;          /* Receive state */
-CONC_WAIT_EVENT wait;          /* Wait-for-completions state */
-CONC_DELAY_EVENT delay;        /* State for spins and sleeps */
-CONC_TOUCH_EVENT touch;        /* State for memory touching */
-CONC_SYNC_EVENT sync;          /* Synchronization state */
-CONC_MCAST_EVENT mcast;        /* Multicast state */
-CONC_REDUCE_EVENT reduce;      /* Reduction state */
-CONC_BTIME_EVENT btime;        /* Timed-loop state */
-CONC_ETIME_EVENT etime;        /* Additional timed-loop state */
-CONC_REPEAT_EVENT rep;         /* Repeated-events state */
-CONC_SUPPRESS_EVENT suppress;  /* State for suppressing output */
-CONC_CODE_EVENT code;          /* State for arbitrary code */
-} s;
-} CONC_EVENT;
-
-/* Fully specify an arbitrary for() loop (used by FOR EACH). */
-typedef struct {
-NCPTL_QUEUE *list_comp;  /* NULL=ordinary list; other=list comprehension values */
-int integral;        /* 1=integral values; 0=floating-point values */
-enum {               /* Comparison of loop variable to end variable */
-CONC_LEQ,                /* Increasing progression */
-CONC_GEQ                 /* Decreasing progression */
-} comparator;
-enum {               /* How to increment the loop variable */
-CONC_ADD,                /* Arithmetically */
-CONC_MULT,               /* Geometrically increasing */
-CONC_DIV                 /* Geometrically decreasing */
-} increment;
-union {
-struct {
-ncptl_int loopvar;   /* Loop variable */
-ncptl_int prev_loopvar; /* Previous value of loop variable */
-ncptl_int startval;  /* Initial value of loop variable */
-ncptl_int endval;    /* Value not to exceed */
-ncptl_int incval;    /* Loop-variable increment */
-} i;
-struct {
-double loopvar;      /* Loop variable */
-double prev_loopvar; /* Previous value of loop variable */
-double startval;     /* Initial value of loop variable */
-double endval;       /* Value not to exceed */
-double incval;       /* Loop-variable increment */
-} d;
-} u;
-} LOOPBOUNDS;
-
-/********************
- * Global variables *
- ********************/
-
-/* Variables exported to coNCePTuaL programs */
-static ncptl_int var_bytes_received = 0;   /* Total number of bytes received */
-static ncptl_int var_msgs_received = 0;   /* Total number of messages received */
-static ncptl_int var_bit_errors = 0;   /* Total number of bit errors observed */
-static ncptl_int var_total_msgs = 0;   /* Sum of messages sent and messages received */
-static ncptl_int var_msgs_sent = 0;   /* Total number of messages sent */
-static ncptl_int var_bytes_sent = 0;   /* Total number of bytes sent */
-static ncptl_int var_num_tasks = 1;   /* Number of tasks running the program */
-static ncptl_int var_elapsed_usecs = 0;   /* Elapsed time in microseconds */
-static ncptl_int var_total_bytes = 0;   /* Sum of bytes sent and bytes received */
-
-/* Dummy variable to help mark other variables as used */
-static union {
-ncptl_int ni;
-int i;
-void *vp;
-} conc_dummy_var;
-
-/* Variables used internally by boilerplate code */
-static uint64_t starttime;   /* Time the clock was last reset (microseconds) */
-static ncptl_int pendingrecvs = 0;   /* Current # of outstanding receives */
-static ncptl_int pendingrecvbytes = 0; /* Current # of bytes in outstanding receives */
-static NCPTL_QUEUE *touchedqueue;      /* Queue of asynchronous receives to touch */
-static ncptl_int pendingsends = 0;   /* Current # of outstanding sends */
-static NCPTL_QUEUE *eventqueue;   /* List of coNCePTuaL events to perform */
-static int within_time_loop = 0;   /* 1=we're within a FOR <time> loop */
-static int suppress_output = 0;    /* 1=suppress output to stdout and the log file */
-static void *touch_region = NULL;   /* Memory region to touch */
-static ncptl_int touch_region_size = 0;   /* # of bytes in the above */
-static int virtrank;    /* This task's virtual rank in the computation */
-static int physrank;    /* This task's physical rank in the computation */
-static NCPTL_VIRT_PHYS_MAP *procmap;  /* Virtual to physical rank mapping */
-static NCPTL_LOG_FILE_STATE *logstate;   /* Opaque object representing all log-file state */
-static char *logfile_uuid;   /* Execution UUID to write to every log file */
-static char *logfiletmpl;   /* Template for the log file's name */
-//static char *logfiletmpl_default;   /* Default value of the above */
-
-/* Global variables specific to the c_mpi backend */
-static ncptl_int mpi_is_running = 0;   /* 1=MPI has been initialized */
-static NCPTL_QUEUE * recvreqQ;   /* List of MPI receive requests */
-static MPI_Request * recvrequests;   /* List version of recvreqQ */
-static NCPTL_QUEUE * recvstatQ;   /* List of MPI receive statuses */
-static MPI_Status * recvstatuses;   /* List version of recvstatQ */
-static NCPTL_QUEUE * sendreqQ;   /* List of MPI send requests */
-static MPI_Request * sendrequests;   /* List version of sendreqQ */
-static NCPTL_QUEUE * sendstatQ;   /* List of MPI send statuses */
-static MPI_Status * sendstatuses;   /* List version of sendstatQ */
-static NCPTL_SET * communicators;   /* Map from an array of processor flags to an MPI communicator */
-static MPI_Errhandler mpi_error_handler;   /* Handle to handle_MPI_error() */
-static ncptl_int mpi_tag_ub;   /* Upper bound on an MPI tag value */
-static ncptl_int conc_mcast_tallies[CONC_MCAST_MPI_NUM_FUNCS] = {0};   /* Tallies of (static) multicast implementation functions */
-
-/* Program-specific variables */
-static ncptl_int var_trials;   /* Number of trials to perform (command-line argument) */
-
-/*************************
- * Function declarations *
- *************************/
-
-/* Make MPI errors invoke ncptl_fatal(). */
-static void handle_MPI_error (MPI_Comm *comm, int *errcode, ...)
-{
-va_list args;
-char errstring[MPI_MAX_ERROR_STRING];
-int errstrlen;
-
-va_start (args, errcode);
-if (MPI_Error_string (*errcode, errstring, &errstrlen) == MPI_SUCCESS)
-ncptl_fatal ("MPI run-time error: %s", errstring);
-else
-ncptl_fatal ("MPI aborted with unrecognized error code %d", *errcode);
-conc_dummy_var.vp = (void *) comm;   /* Prevent the compiler from complaining that comm is unused. */
-va_end (args);
-}
-
-/* Perform the equivalent of CODES_MPI_Comm_rank() for an arbitrary process. */
-static int rank_in_MPI_communicator (MPI_Comm subcomm, int global_rank)
-{
-  MPI_Group world_group;   /* Group associated with MPI_COMM_WORLD */
-  MPI_Group subgroup;      /* Group associate with subcomm */
-  int subrank;             /* global_rank's rank within subcomm */
-
-  MPI_Comm_group (MPI_COMM_WORLD, &world_group);
-  MPI_Comm_group (subcomm, &subgroup);
-  MPI_Group_translate_ranks (world_group, 1, &global_rank, subgroup, &subrank);
-  return subrank;
-}
-
-/* Map an arbitrary tag to within MPI's valid range of [0, mpi_tag_ub]. */
-static ncptl_int map_tag_into_MPI_range (ncptl_int tag)
-{
-if (tag == NCPTL_INT_MIN)
- /* Avoid taking the absolute value of NCPTL_INT_MIN. */
-tag = 555666773LL;   /* Arbitrary value */
-tag = ncptl_func_abs (tag);   /* Only nonnegatives values are allowed. */
-if (mpi_tag_ub < NCPTL_INT_MAX)
-tag %= mpi_tag_ub + 1;
-return tag;
-}
-
-/* Given an array of task in/out booleans return an MPI
- * communicator that represents the "in" tasks. */
-static MPI_Comm define_MPI_communicator (char *procflags)
-{
-MPI_Comm *existing_comm;    /* Previously defined MPI communicator */
-MPI_Comm new_comm;          /* Newly defined MPI communicator */
-
-existing_comm = (MPI_Comm *) ncptl_set_find (communicators, (void *)procflags);
-if (existing_comm)
-return *existing_comm;
-(void) MPI_Comm_split (MPI_COMM_WORLD, (int)procflags[physrank], physrank, &new_comm);
-//(void) MPI_Errhandler_set (new_comm, mpi_error_handler);
-ncptl_set_insert (communicators, (void *)procflags, (void *)&new_comm);
-return define_MPI_communicator (procflags);
-}
-
-/* Return 1 if a sequence loop will take at least one trip. */
-static int conc_seq_nonempty (LOOPBOUNDS *seq)
-{
-ncptl_int startval;   /* Integer version of seq's startval element */
-ncptl_int endval;   /* Integer version of seq's endval element */
-
-if (seq->integral) {
-startval = seq->u.i.startval;
-endval = seq->u.i.endval;
-}
-else {
-startval = CONC_DBL2INT (seq->u.d.startval);
-endval = CONC_DBL2INT (seq->u.d.endval);
-}
-switch (seq->comparator) {
-case CONC_LEQ:
-return startval <= endval;
-
-case CONC_GEQ:
-return startval >= endval;
-
-default:
-ncptl_fatal ("Internal error -- unknown comparator");
-}
-return -1;     /* Appease idiotic compilers. */
-}
-
-/* Initialize a sequence loop. */
-static void conc_seq_init (LOOPBOUNDS *seq)
-{
-if (seq->integral) {
-seq->u.i.loopvar = seq->u.i.startval;
-seq->u.i.prev_loopvar = seq->u.i.loopvar - 1;
-}
-else {
-seq->u.d.loopvar = seq->u.d.startval;
-seq->u.d.prev_loopvar = seq->u.d.loopvar - 1.0;
-}
-}
-
-/* Return 1 if a sequence loop should continue, 0 when finished. */
-static int conc_seq_continue (LOOPBOUNDS *seq)
-{
-LOOPBOUNDS seq_int;   /* Integer equivalent of *seq */
-if (seq->integral)
-seq_int = *seq;
-else {
-seq_int.u.i.loopvar = CONC_DBL2INT (seq->u.d.loopvar);
-seq_int.u.i.prev_loopvar = CONC_DBL2INT (seq->u.d.prev_loopvar);
-seq_int.u.i.endval = CONC_DBL2INT (seq->u.d.endval);
-}
-
-if (seq_int.u.i.loopvar == seq_int.u.i.prev_loopvar)
-return 0;
-switch (seq->comparator) {
-case CONC_LEQ:
-return seq_int.u.i.loopvar <= seq_int.u.i.endval;
-
-case CONC_GEQ:
-return seq_int.u.i.loopvar >= seq_int.u.i.endval;
-
-default:
-ncptl_fatal ("Internal error -- unknown comparator");
-}
-return -1;     /* Appease idiotic compilers. */
-}
-
-/* Proceed to the next iteration of a sequence loop. */
-static void conc_seq_next (LOOPBOUNDS *seq)
-{
-if (seq->integral) {
-seq->u.i.prev_loopvar = seq->u.i.loopvar;
-switch (seq->increment) {
-case CONC_ADD:
-seq->u.i.loopvar += seq->u.i.incval;
-break;
-
-case CONC_MULT:
-seq->u.i.loopvar *= seq->u.i.incval;
-break;
-
-case CONC_DIV:
-seq->u.i.loopvar /= seq->u.i.incval;
-break;
-
-default:
-ncptl_fatal ("Internal error -- unknown incrementer");
-}
-}
-else {
-seq->u.d.prev_loopvar = seq->u.d.loopvar;
-switch (seq->increment) {
-case CONC_ADD:
-seq->u.d.loopvar += seq->u.d.incval;
-break;
-
-case CONC_MULT:
-seq->u.d.loopvar *= seq->u.d.incval;
-break;
-
-case CONC_DIV:
-seq->u.d.loopvar /= seq->u.d.incval;
-break;
-
-default:
-ncptl_fatal ("Internal error -- unknown incrementer");
-}
-}
-}
-
-/* Inhibit the compiler from complaining that
- * certain variables are defined but not used.
- * This function should never be called. */
-static void conc_mark_variables_used (void)
-{
-conc_dummy_var.ni = var_bytes_received;
-conc_dummy_var.ni = var_msgs_received;
-conc_dummy_var.ni = var_bit_errors;
-conc_dummy_var.ni = var_total_msgs;
-conc_dummy_var.ni = var_msgs_sent;
-conc_dummy_var.ni = var_bytes_sent;
-conc_dummy_var.ni = var_num_tasks;
-conc_dummy_var.ni = var_elapsed_usecs;
-conc_dummy_var.ni = var_total_bytes;
-conc_dummy_var.ni = pendingrecvbytes;
-conc_dummy_var.ni = touch_region_size;
-conc_dummy_var.vp = touch_region;
-conc_dummy_var.i  = within_time_loop;
-conc_dummy_var.i  = suppress_output;
-rank_in_MPI_communicator (MPI_COMM_WORLD, 0);
-}
-
-/* Allocate a new event of a given type and return a pointer to it. */
-static CONC_EVENT *conc_allocate_event (CONC_EVENT_TYPE type)
-{
-CONC_EVENT *newevent = (CONC_EVENT *) ncptl_queue_allocate (eventqueue);
-
-newevent->type = type;
-return newevent;
-}
-
-/* Declare an exit handler that gets called automatically when the
- * program terminates, whether successfully or not. */
-static void conc_exit_handler (void)
-{
-if (mpi_is_running)
-MPI_Abort (MPI_COMM_WORLD, 1);
-}
-
-
-/* Initialize coNCePTuaL, the messaging layer, and this program itself. */
-static void conc_initialize (int argc, char *argv[])
-{
- /* Variables needed by all C-based backends */
-CONC_EVENT * eventlist;   /* List of events to execute */
-ncptl_int numevents;   /* Number of entries in eventlist[] */
-int help_only = 0;   /* 1=User specified --help; save time by skipping ncptl_init() */
-char * argv0 = strrchr(argv[0], '/') ? strrchr(argv[0], '/')+1 : argv[0];   /* Base name of the executable program */
-int i;   /* Generic loop variable */
-
- /* Declare all of our command-line arguments. */
-NCPTL_CMDLINE arguments[] = {
-{ NCPTL_TYPE_STRING, NULL, "logfile", 'L', "Log-file template", {0}},
-{ NCPTL_TYPE_INT, NULL, "trials", 't', "Number of trials to perform", {0}}
-};
-
- /* Incorporate the complete coNCePTuaL source code as an array
-//  * for use by ncptl_log_write_prologue(). */
-char *sourcecode[] = {
-"# Virtual ring \"hot potato\" test",
-"",
-"Require language version \"1.5\".",
-"",
-"trials is \"Number of trials to perform\" and comes from \"--trials\" or",
-"  \"-t\" with default 100000.",
-"",
-"Assert that \"the hot-potato test requires at least two tasks\" with num_tasks>=2.",
-"",
-"Let len be 0 while {",
-"  for each task_count in {2, ..., num_tasks} {",
-"    task 0 outputs \"Performing \" and trials and \" \" and",
-"		   task_count and \"-task runs....\" then",
-"    for trials repetitions plus 5 warmup repetitions and a synchronization {",
-"      task 0 resets its counters then",
-"      task 0 sends a len byte message to unsuspecting task 1 then",
-"      task (n+1) mod task_count receives a len byte message from task n such that n<task_count then",
-"      task n such that n>0 /\\ n<task_count sends a len byte message to unsuspecting task (n+1) mod task_count then",
-"      task 0 logs the task_count as \"# of tasks\" and",
-"		  the minimum of elapsed_usecs/task_count as \"Latency (usecs)\" and",
-"		  the mean of elapsed_usecs/task_count as \"Latency (usecs)\" and",
-"		  the variance of elapsed_usecs/task_count as \"Latency (usecs)\"",
-"    } then",
-"    task 0 computes aggregates",
-"  }",
-"}",
-NULL
-};
-
- /* Variables specific to the c_mpi backend */
-int num_tasks;   /* int version of var_num_tasks needed by CODES_MPI_Comm_size() */
-char * procflags;   /* Array of 1s representing an all-task MPI communicator */
-MPI_Comm comm_world = MPI_COMM_WORLD;   /* Copy of MPI_COMM_WORLD that we can take the address of */
-void * attr_val;   /* Pointed to the value of MPI_TAG_UB */
-int attr_flag = 0;   /* true=MPI_TAG_UB was extracted; false=not extracted */
-char log_key_str[128];   /* String representing the range of valid MPI tags */
-
- /* As a special case, if the command line contains --help, then skip
-  * the coNCePTuaL initialization step. */
-for (i=1; i<argc; i++)
-if (!strcmp(argv[i], "--"))
-break;
-else
-if (!strcmp(argv[i], "--help") || !strcmp(argv[i], "-?")) {
-argv[1] = "-?";   /* Guaranteed to work, even with getopt() */
-help_only = 1;
-break;
-}
-
- /* Perform various initializations specific to the c_mpi backend. */
- /* Initialize MPI. */
-//(void) MPI_Init(&argc, &argv);
-mpi_is_running = 1;
-
- /* Initialize the coNCePTuaL run-time library. */
-if (!help_only)
-//ncptl_init (NCPTL_RUN_TIME_VERSION, argv[0]);
-//(void) atexit (conc_exit_handler);
-
- /* Initialize the communication routines needed by the c_mpi backend. */
-//(void) MPI_Errhandler_create ((MPI_Handler_function *)handle_MPI_error, &mpi_error_handler);
-//(void) MPI_Errhandler_set (MPI_COMM_WORLD, mpi_error_handler);
-(void) CODES_MPI_Comm_rank(MPI_COMM_WORLD, &physrank);
-(void) CODES_MPI_Comm_size(MPI_COMM_WORLD, &num_tasks);
-var_num_tasks = (ncptl_int) num_tasks;
-//(void) MPI_Comm_get_attr(MPI_COMM_WORLD, MPI_TAG_UB, &attr_val, &attr_flag);
-mpi_tag_ub = (ncptl_int) (attr_flag ? *(int *)attr_val : 32767);
-
- /* Generate and broadcast a UUID. */
-//logfile_uuid = ncptl_log_generate_uuid();
-//(void) CODES_MPI_Bcast ((void *)logfile_uuid, 37, MPI_CHAR, 0, MPI_COMM_WORLD);
-
- /* Plug variables and default values into the NCPTL_CMDLINE structure. */
-arguments[0].variable = (CMDLINE_VALUE *) &logfiletmpl;
-arguments[1].variable = (CMDLINE_VALUE *) &var_trials;
-arguments[1].defaultvalue.intval = 100000LL;
-//logfiletmpl_default = (char *) ncptl_malloc (strlen(argv0) + 15, 0);
-//sprintf (logfiletmpl_default, "%s-%%p.log", argv0);
-//arguments[0].defaultvalue.stringval = logfiletmpl_default;
-
- /* Parse the command line. */
-mpi_is_running = 0;   /* Don't invoke MPI_Abort() after --help. */
-ncptl_parse_command_line (argc, argv, arguments, sizeof(arguments)/sizeof(NCPTL_CMDLINE));
-if (help_only)
-ncptl_fatal ("Internal error in the c_generic backend: failed to exit after giving help");
-mpi_is_running = 1;
-
- /* Establish a mapping from (virtual) task IDs to (physical) ranks. */
-procmap = ncptl_allocate_task_map (var_num_tasks);
-virtrank = ncptl_physical_to_virtual (procmap, physrank);
-
- /* Perform initializations specific to the c_mpi backend. */
-//ncptl_log_add_comment ("MPI send routines", "MPI_Send() and CODES_MPI_Isend()");
-//ncptl_log_add_comment ("MPI reduction operation", REDUCE_OPERATION_NAME);
-//sprintf (log_key_str, "[0, %" NICS "]", mpi_tag_ub);
-//ncptl_log_add_comment ("MPI tag range", log_key_str);
-//
-// /* Open the log file and write some standard prologue information to it. */
-//logstate = ncptl_log_open (logfiletmpl, physrank);
-////ncptl_log_write_prologue (logstate, argv[0], logfile_uuid, "c_mpi", "C + MPI",
-//var_num_tasks,
-//arguments, sizeof(arguments)/sizeof(NCPTL_CMDLINE),
-//sourcecode);
-//ncptl_free (logfile_uuid);
-
- /* Allocate a variety of dynamically growing queues. */
-eventqueue = ncptl_queue_init (sizeof (CONC_EVENT));
-touchedqueue = ncptl_queue_init (sizeof (ncptl_int));
-
- /* Perform initializations specific to the c_mpi backend. */
-sendreqQ = ncptl_queue_init (sizeof (MPI_Request));
-sendstatQ = ncptl_queue_init (sizeof (MPI_Status));
-recvreqQ = ncptl_queue_init (sizeof (MPI_Request));
-recvstatQ = ncptl_queue_init (sizeof (MPI_Status));
-communicators = ncptl_set_init (ESTIMATED_COMMUNICATORS, var_num_tasks*sizeof(char), sizeof(MPI_Comm));procflags = (char *) ncptl_malloc (var_num_tasks*sizeof(char), 0);
-for (i=0; i<var_num_tasks; i++)
-procflags[i] = 1;
-ncptl_set_insert (communicators, (void *)procflags, (void *)&comm_world);
-ncptl_free (procflags);
-
- /****************************************************
-  * Generated, program-specific initialization code. *
-  ****************************************************/
-
- /* ASSERT THAT the hot-potato test requires at least two tasks. */
-if (!((var_num_tasks)>=(2LL)))
-ncptl_fatal ("Assertion failure: the hot-potato test requires at least two tasks");
-{
- /* LET var_len BE 0LL WHILE... */
-ncptl_int var_len = 0LL;
-
- /* FOR EACH var_task_count IN [(['2LL'], 'var_num_tasks')]... */
-{
-LOOPBOUNDS loopbounds[1];   /* List of range descriptions */
-ncptl_int rangenum;   /* Current offset into loopbounds[] */
-ncptl_int initial_vals[1];   /* Cache of the initial, enumerated values */
-ncptl_int final_val;   /* Cache of the final value */
-
- /* Write range 0's loop bounds, "next" function, and termination function to loopbounds[0]. */
-loopbounds[0].list_comp = NULL;
-initial_vals[0] = 2LL;
-final_val = var_num_tasks;
-loopbounds[0].integral = 1;
-loopbounds[0].u.i.startval = initial_vals[0];
-loopbounds[0].u.i.endval = final_val;
-loopbounds[0].comparator = initial_vals[0]<=final_val ? CONC_LEQ : CONC_GEQ;
-loopbounds[0].increment = CONC_ADD;
-loopbounds[0].u.i.incval = initial_vals[0]<=final_val ? 1LL : -1LL;
-
- /* Now that we've defined all of our ranges we iterate over each range
-  * and each element within each range. */
-for (rangenum=0LL; rangenum<1; rangenum++) {
-LOOPBOUNDS * thisrange = &loopbounds[rangenum];   /* Current range */
-if (conc_seq_nonempty (thisrange))
-for (conc_seq_init (thisrange);
-conc_seq_continue (thisrange);
-conc_seq_next (thisrange)) {
-ncptl_int var_task_count;
-if (thisrange->list_comp == NULL)
- /* The loopvar field contains the value for var_task_count. */
-var_task_count = thisrange->integral ? thisrange->u.i.loopvar : CONC_DBL2INT(thisrange->u.d.loopvar);
-else
- /* The queue of list-comprehension values provides the value for var_task_count. */
-var_task_count = *(ncptl_int *)ncptl_queue_pop(thisrange->list_comp);
-{
- /* TASK 0LL OUTPUTS ['"Performing "', 'var_trials', '" "', 'var_task_count', '"-task runs...."'] */
-if ((0LL) == virtrank) {   /* TASK 0LL */
- /* The current coNCePTuaL statement applies to our task. */
-CONC_EVENT *thisev = conc_allocate_event (EV_CODE);
-thisev->s.code.number = 0;
-thisev->s.code.procmap = NULL;
-thisev->s.code.var_task_count = var_task_count;
-thisev->s.code.var_trials = var_trials;
-}
- /* THEN... */
- /* FOR var_trials REPETITIONS... */
-{
-ncptl_int loopnum;   /* 0=warmup repetitions; 1=main repetitions */
-
-for (loopnum=0LL; loopnum<2LL; loopnum++) {
-ncptl_int numreps = loopnum==0LL ? (5LL) : (var_trials);   /* Total # of repetitions to perform */
-int unroll_loop = numreps <= CONC_MAX_UNROLL;   /* 1=unroll loop; 0=use a REPEAT event */
-ncptl_int repnum;   /* Current repetition number */
-CONC_EVENT * repevent = NULL;   /* Event designating repetition */
-ncptl_int repeventnum = -1LL;   /* Event number corresponding to repevent */
-ncptl_int on_event = -1;   /* Event ID of the "suppression on" event */
-
- /* Suppress output at the start of the warmup repetitions. */
-if (numreps > 0LL && loopnum == 0LL) {
-CONC_EVENT * thisev_on;   /* Output-suppression event */
-thisev_on =  conc_allocate_event (EV_SUPPRESS);
-thisev_on->s.suppress.quiet = 1;
-on_event = ncptl_queue_length(eventqueue) - 1;
-}
-
- /* Conditionally unroll the loop. */
-for (repnum=0; repnum<(unroll_loop?numreps:1); repnum++) {
- /* Allocate a repeat event if we have more than one repetition. */
-if (!unroll_loop && numreps > 1LL) {
-repeventnum = ncptl_queue_length (eventqueue);
-repevent =  conc_allocate_event (EV_REPEAT);
-repevent->s.rep.numreps = numreps;
-}
-
- /* Output a loop body if we have at least one repetition. */
-if (unroll_loop || numreps > 0LL) {
- /* TASK 0LL RESET THEIR COUNTERS */
-if ((0LL) == virtrank) {   /* TASK 0LL */
- /* The current coNCePTuaL statement applies to our task. */
-(void) conc_allocate_event (EV_RESET);
-}
- /* THEN... */
- /* TASK 0LL SENDS TO TASK 1LL */
-if ((0LL) == virtrank) {   /* TASK 0LL */
- /* The current coNCePTuaL statement applies to our task. */
-ncptl_int virtdest = 1LL;
-if (virtdest>=0 && virtdest<var_num_tasks) {
- /* In this scope, 1LL represents a single receiver. */
- /* Prepare to send 1LL messages. */
-ncptl_int numreps = 1LL;   /* Number of messages */
-if (numreps > 1LL) {
-CONC_EVENT * repeatev;   /* Event specifying the number of repetitions to perform */
-repeatev = conc_allocate_event (EV_REPEAT);
-repeatev->s.rep.end_event = ncptl_queue_length(eventqueue);
-repeatev->s.rep.numreps = numreps;
-}
-
- /* Ensure we have at least one message to send. */
-if (numreps > 0LL) {
-CONC_EVENT *thisev = conc_allocate_event (EV_SEND);
-
- /* Fill in all of the fields of a send-event structure. */
-thisev->s.send.dest = ncptl_virtual_to_physical (procmap, 1LL);
-thisev->s.send.size = var_len;
-thisev->s.send.alignment = 0LL;
-thisev->s.send.misaligned = 0;
-thisev->s.send.touching = 0;
-thisev->s.send.verification = 0;
-thisev->s.send.tag = 0LL;
-thisev->s.send.pendingsends = pendingsends;
-thisev->s.send.pendingrecvs = pendingrecvs;
-thisev->s.send.bufferofs = 0LL;
-thisev->s.send.buffernum = thisev->s.send.verification ? pendingsends+pendingrecvs : 0;
-(void) ncptl_malloc_message (thisev->s.send.size+thisev->s.send.bufferofs, thisev->s.send.alignment, thisev->s.send.buffernum, thisev->s.send.misaligned);
-thisev->s.send.buffer = NULL;
-thisev->s.send.tag = map_tag_into_MPI_range (thisev->s.send.tag);
-}
-}
-}
- /* THEN... */
- /* TASKS var_n SUCH THAT (var_n)<(var_task_count) RECEIVES FROM TASK ncptl_func_modulo((var_n)+(1LL),var_task_count) */
-{
-ncptl_int var_n;
- /* Loop over all tasks to see which will send to us. */
-for (var_n=0; var_n<var_num_tasks; var_n++)
-if ((var_n)<(var_task_count)) {
- /* var_n now represents one of the tasks that will send to us. */
-ncptl_int ivar_a_task = var_n;
-if (virtrank == (ncptl_func_modulo((var_n)+(1LL),var_task_count))) {
- /* In this scope, we must be a message recipient. */
- /* Prepare to receive 1LL messages. */
-ncptl_int numreps = 1LL;   /* Number of messages */
-if (numreps > 1LL) {
-CONC_EVENT * repeatev;   /* Event specifying the number of repetitions to perform */
-repeatev = conc_allocate_event (EV_REPEAT);
-repeatev->s.rep.end_event = ncptl_queue_length(eventqueue);
-repeatev->s.rep.numreps = numreps;
-}
-
- /* Ensure we have at least one message to receive. */
-if (numreps > 0LL) {
-CONC_EVENT *thisev = conc_allocate_event (EV_RECV);
-
- /* Fill in all of the fields of a receive-event structure. */
-thisev->s.recv.source = ncptl_virtual_to_physical (procmap, ivar_a_task);
-thisev->s.recv.size = var_len;
-thisev->s.recv.alignment = 0LL;
-thisev->s.recv.misaligned = 0;
-thisev->s.recv.touching = 0;
-thisev->s.recv.verification = 0;
-thisev->s.recv.tag = 0LL;
-thisev->s.recv.pendingsends = pendingsends;
-thisev->s.recv.pendingrecvs = pendingrecvs;
-thisev->s.recv.bufferofs = 0LL;
-thisev->s.recv.buffernum = thisev->s.recv.verification ? pendingsends+pendingrecvs : 0;
-(void) ncptl_malloc_message (thisev->s.recv.size+thisev->s.recv.bufferofs, thisev->s.recv.alignment, thisev->s.recv.buffernum, thisev->s.recv.misaligned);
-thisev->s.recv.buffer = NULL;
-thisev->s.recv.tag = map_tag_into_MPI_range (thisev->s.recv.tag);
-}
-}
-}
-}
- /* THEN... */
- /* TASKS var_n SUCH THAT ((var_n)>(0LL)) && ((var_n)<(var_task_count)) SENDS TO TASK ncptl_func_modulo((var_n)+(1LL),var_task_count) */
-{
-ncptl_int var_n = virtrank;
-if (((var_n)>(0LL)) && ((var_n)<(var_task_count))) {   /* TASKS var_n SUCH THAT ((var_n)>(0LL)) && ((var_n)<(var_task_count)) */
- /* The current coNCePTuaL statement applies to our task. */
-ncptl_int virtdest = ncptl_func_modulo((var_n)+(1LL),var_task_count);
-if (virtdest>=0 && virtdest<var_num_tasks) {
- /* In this scope, ncptl_func_modulo((var_n)+(1LL),var_task_count) represents a single receiver. */
- /* Prepare to send 1LL messages. */
-ncptl_int numreps = 1LL;   /* Number of messages */
-if (numreps > 1LL) {
-CONC_EVENT * repeatev;   /* Event specifying the number of repetitions to perform */
-repeatev = conc_allocate_event (EV_REPEAT);
-repeatev->s.rep.end_event = ncptl_queue_length(eventqueue);
-repeatev->s.rep.numreps = numreps;
-}
-
- /* Ensure we have at least one message to send. */
-if (numreps > 0LL) {
-CONC_EVENT *thisev = conc_allocate_event (EV_SEND);
-
- /* Fill in all of the fields of a send-event structure. */
-thisev->s.send.dest = ncptl_virtual_to_physical (procmap, ncptl_func_modulo((var_n)+(1LL),var_task_count));
-thisev->s.send.size = var_len;
-thisev->s.send.alignment = 0LL;
-thisev->s.send.misaligned = 0;
-thisev->s.send.touching = 0;
-thisev->s.send.verification = 0;
-thisev->s.send.tag = 0LL;
-thisev->s.send.pendingsends = pendingsends;
-thisev->s.send.pendingrecvs = pendingrecvs;
-thisev->s.send.bufferofs = 0LL;
-thisev->s.send.buffernum = thisev->s.send.verification ? pendingsends+pendingrecvs : 0;
-(void) ncptl_malloc_message (thisev->s.send.size+thisev->s.send.bufferofs, thisev->s.send.alignment, thisev->s.send.buffernum, thisev->s.send.misaligned);
-thisev->s.send.buffer = NULL;
-thisev->s.send.tag = map_tag_into_MPI_range (thisev->s.send.tag);
-}
-}
-}
-}
- /* THEN... */
- /* TASK 0LL LOGS "# of tasks" AND "Latency (usecs)" AND "Latency (usecs)" AND "Latency (usecs)" */
-if ((0LL) == virtrank) {   /* TASK 0LL */
- /* The current coNCePTuaL statement applies to our task. */
-CONC_EVENT *thisev = conc_allocate_event (EV_CODE);
-thisev->s.code.number = 1;
-thisev->s.code.procmap = NULL;
-thisev->s.code.var_task_count = var_task_count;
-}
-}
-}
-
- /* Assign the number of events to repeat, now that we know that number. */
-if (!unroll_loop && numreps > 1LL) {
-repevent = repeventnum + (CONC_EVENT *) ncptl_queue_contents (eventqueue, 0);
-repevent->s.rep.end_event = ncptl_queue_length (eventqueue) - 1;
-}
-
- /* After the warmups are finished, synchronize all tasks. */
-if (loopnum == 0LL) {
-CONC_EVENT * thisev_sync;   /* Barrier-synchronization event */
-
-thisev_sync =  conc_allocate_event (EV_SYNC);
-thisev_sync->s.sync.communicator = MPI_COMM_WORLD;
-}
-
- /* After the warmups are finished, resume outputting. */
-if (numreps > 0LL && loopnum == 0LL) {
-CONC_EVENT * thisev_off;   /* Output un-suppression event */
-
-thisev_off =  conc_allocate_event (EV_SUPPRESS);
-thisev_off->s.suppress.quiet = 0;
-thisev_off->s.suppress.matching_event = on_event;
-}
-}
-}
- /* THEN... */
- /* TASK 0LL COMPUTES AGGREGATES */
-if ((0LL) == virtrank) {   /* TASK 0LL */
- /* The current coNCePTuaL statement applies to our task. */
-(void) conc_allocate_event (EV_FLUSH);
-}
-}
-}
-}
-}
-}
- /* Begin a new top-level statement. */
-(void) conc_allocate_event (EV_NEWSTMT);
-
- /*************************
-  * More boilerplate code *
-  *************************/
-
- /* Abort if the program will terminate with pending messages. */
-if (pendingsends && pendingrecvs)
-ncptl_fatal("Neglected to await the completion of %" NICS " asynchronous %s and %" NICS " asynchronous %s",
-pendingsends, pendingsends==1LL ? "send" : "sends",
-pendingrecvs, pendingrecvs==1LL ? "receive" : "receives");
-else
-if (pendingsends)
-ncptl_fatal("Neglected to await the completion of %" NICS " asynchronous %s",
-pendingsends, pendingsends==1LL ? "send" : "sends");
-else
-if (pendingrecvs)
-ncptl_fatal("Neglected to await the completion of %" NICS " asynchronous %s",
-pendingrecvs, pendingrecvs==1LL ? "receive" : "receives");
-
- /* Allocate memory for non-unique messages and asynchronous
-  * message handles now that we know how much memory we need
-  * to allocate. */
-eventlist = (CONC_EVENT *) ncptl_queue_contents (eventqueue, 0);
-numevents = ncptl_queue_length (eventqueue);
-sendrequests = (MPI_Request *) ncptl_queue_contents (sendreqQ, 0);
-recvrequests = (MPI_Request *) ncptl_queue_contents (recvreqQ, 0);
-for (i=0; i<numevents; i++) {
-CONC_EVENT *thisev = &eventlist[i];   /* Cache of the current event */
-switch (thisev->type) {
-case EV_SEND:
-if (!thisev->s.send.buffer)
-thisev->s.send.buffer = ncptl_malloc_message (thisev->s.send.bufferofs + thisev->s.send.size,
-thisev->s.send.alignment,
-thisev->s.send.buffernum,
-thisev->s.send.misaligned);
-if (thisev->s.send.verification)
-ncptl_fill_buffer (CONC_GETBUFPTR(send), thisev->s.send.bufferofs + thisev->s.send.size, -1);
-break;
-
-case EV_RECV:
-if (!thisev->s.recv.buffer)
-thisev->s.recv.buffer = ncptl_malloc_message (thisev->s.recv.bufferofs + thisev->s.recv.size,
-thisev->s.recv.alignment,
-thisev->s.recv.buffernum,
-thisev->s.recv.misaligned);
-if (thisev->s.recv.verification)
-ncptl_fill_buffer (CONC_GETBUFPTR(recv), thisev->s.recv.bufferofs + thisev->s.recv.size, -1);
-break;
-
-default:
-break;
-}
-}
-}
-
-/* Process a subset of the events in a given event list. */
-static void conc_process_events (CONC_EVENT *eventlist,
-ncptl_int firstev, ncptl_int lastev, ncptl_int numreps)
-{
-CONC_EVENT * thisev;   /* Cache of the current event */
-CONC_EVENT * thisev_first = &eventlist[firstev];   /* Cache of the first event */
-ncptl_int i;   /* Iterate over events. */
-ncptl_int j;   /* Iterate over repetitions. */
-
- /* Declarations specific to the c_mpi backend */
-MPI_Status status;   /* Not needed but required by CODES_MPI_Recv() */
-
- /* Process from event firstev to event lastev (both inclusive). */
-for (j=numreps; j>0; j--)
-for (i=firstev, thisev=thisev_first; i<=lastev; i++, thisev++) {
- /* Declare variables needed by all C-based backends. */
-
- /* Process a single event. */
-switch (thisev->type) {
-case EV_SEND:
- /* Synchronous send */
-(void) CODES_MPI_Send (CONC_GETBUFPTR(send),
-(int)thisev->s.send.size, MPI_BYTE,
-(int)thisev->s.send.dest, (int)thisev->s.send.tag, MPI_COMM_WORLD);
-break;
-
-case EV_RECV:
- /* Synchronous receive */
-(void) CODES_MPI_Recv (CONC_GETBUFPTR(recv),
-(int)thisev->s.recv.size, MPI_BYTE,
-(int)thisev->s.recv.source, (int)thisev->s.recv.tag,
-MPI_COMM_WORLD, &status);
-break;
-
-case EV_SYNC:
- /* Synchronize a subset of the tasks. */
-(void) CODES_MPI_Barrier (thisev->s.sync.communicator);
-break;
-
-case EV_RESET:
- /* Reset all of the counters exported to coNCePTuaL programs. */
-starttime = ncptl_time();
-break;
-
-case EV_FLUSH:
- /* Force all aggregate functions to produce a result. */
-if (!suppress_output) {
-uint64_t stop_elapsed_usecs = ncptl_time();
-var_elapsed_usecs = stop_elapsed_usecs - starttime;
-//ncptl_log_compute_aggregates (logstate);
-starttime += ncptl_time() - stop_elapsed_usecs;
-}
-break;
-
-case EV_SUPPRESS:
- /* Turn logging and outputting on or off. */
-if (thisev->s.suppress.quiet) {
- /* Suppress output. */
-thisev->s.suppress.prev_quiet = suppress_output;
-suppress_output = 1;
-thisev->s.suppress.stop_elapsed_usecs = ncptl_time();
-}
-else {
- /* Restore suppression to its previous state. */
-CONC_SUPPRESS_EVENT * matchev = &eventlist[thisev->s.suppress.matching_event].s.suppress;   /* Matching suppression event */
-suppress_output = matchev->prev_quiet;
-if (!suppress_output) {
-starttime += ncptl_time() - matchev->stop_elapsed_usecs;
-}
-}
-break;
-
-case EV_REPEAT:
- /* Repeatedly perform the next batch of events. */
-conc_process_events (eventlist, i+1, thisev->s.rep.end_event, thisev->s.rep.numreps);
-i = thisev->s.rep.end_event;
-thisev = &eventlist[i];
-break;
-
-case EV_NEWSTMT:
- /* Begin a new table in the log file. */
-if (!suppress_output) {
-uint64_t stop_elapsed_usecs = ncptl_time();
-var_elapsed_usecs = stop_elapsed_usecs - starttime;
-//ncptl_log_commit_data (logstate);
-starttime += ncptl_time() - stop_elapsed_usecs;
-}
-break;
-
-case EV_CODE:
- /* Execute an arbitrary piece of code. */
-switch (thisev->s.code.number) {
-case 0:
- /* TASK 0LL OUTPUTS ['"Performing "', 'var_trials', '" "', 'var_task_count', '"-task runs...."'] */
-if (!suppress_output) {
-uint64_t stop_elapsed_usecs = ncptl_time();
-var_elapsed_usecs = stop_elapsed_usecs - starttime;
-printf ("%s%.10lg%s%.10lg%s\n", "Performing ", (double)thisev->s.code.var_trials, " ", (double)thisev->s.code.var_task_count, "-task runs....");
-fflush (stdout);
-starttime += ncptl_time() - stop_elapsed_usecs;
-}
-break;
-
-case 1:
- /* TASK 0LL LOGS "# of tasks" AND "Latency (usecs)" AND "Latency (usecs)" AND "Latency (usecs)" */
-if (!suppress_output) {
-uint64_t stop_elapsed_usecs = ncptl_time();
-var_elapsed_usecs = stop_elapsed_usecs - starttime;
-//ncptl_log_write (logstate, 0, "# of tasks", NCPTL_FUNC_ONLY, 0.0, (double)thisev->s.code.var_task_count);
-//ncptl_log_write (logstate, 1, "Latency (usecs)", NCPTL_FUNC_MINIMUM, 0.0, ((double)var_elapsed_usecs)/((double)thisev->s.code.var_task_count));
-//ncptl_log_write (logstate, 2, "Latency (usecs)", NCPTL_FUNC_MEAN, 0.0, ((double)var_elapsed_usecs)/((double)thisev->s.code.var_task_count));
-//ncptl_log_write (logstate, 3, "Latency (usecs)", NCPTL_FUNC_VARIANCE, 0.0, ((double)var_elapsed_usecs)/((double)thisev->s.code.var_task_count));
-starttime += ncptl_time() - stop_elapsed_usecs;
-}
-break;
-
-default:
- /* The C code generation module must be broken. */
-ncptl_fatal ("Internal error: unknown EV_CODE block %" NICS, thisev->s.code.number);
-break;
-}
-break;
-
-default:
- /* The c_generic backend or the c_mpi backend must be broken. */
-ncptl_fatal ("Internal error: unknown event type %d", thisev->type);
-break;
-}
-}
-}
-
-/* Finish up cleanly and return a status code. */
-static int conc_finalize (void)
-{
-int exitcode = 0;   /* Program exit code (to pass to exit()) */
-
- /* Declarations specific to the c_mpi backend */
-//int mpiresult;   /* Return code from MPI_Finalize() */
-//char log_key_str[128];   /* String representing the range of valid MPI tags */
-//
-// /* Finalization code specific to the c_mpi backend */
-//log_key_str[0] = '\0';
-//if (conc_mcast_tallies[CONC_MCAST_MPI_BCAST] > 0) {
-//char onefuncstr[50];
-//sprintf (onefuncstr, "%sCODES_MPI_Bcast()*%" NICS,
-//log_key_str[0] == '\0' ? "" : " ", conc_mcast_tallies[CONC_MCAST_MPI_BCAST]);
-//strcat (log_key_str, onefuncstr);
-//}
-//if (conc_mcast_tallies[CONC_MCAST_MPI_ALLTOALL] > 0) {
-//char onefuncstr[50];
-//sprintf (onefuncstr, "%sCODES_MPI_Alltoall()*%" NICS,
-//log_key_str[0] == '\0' ? "" : " ", conc_mcast_tallies[CONC_MCAST_MPI_ALLTOALL]);
-//strcat (log_key_str, onefuncstr);
-//}
-//if (conc_mcast_tallies[CONC_MCAST_MPI_ALLTOALLV] > 0) {
-//char onefuncstr[50];
-//sprintf (onefuncstr, "%sCODES_MPI_Alltoallv()*%" NICS,
-//log_key_str[0] == '\0' ? "" : " ", conc_mcast_tallies[CONC_MCAST_MPI_ALLTOALLV]);
-//strcat (log_key_str, onefuncstr);
-//}
-//if (log_key_str[0] != '\0')
-//ncptl_log_add_comment ("Multicast functions used (statically)", log_key_str);
-//
-// /* Write a standard epilogue to the log file. */
-////ncptl_log_commit_data (logstate);
-////ncptl_log_write_epilogue (logstate);
-//ncptl_log_close (logstate);
-
- /* Inform the run-time library that it's no longer needed. */
-//ncptl_queue_empty (eventqueue);
-//ncptl_free (eventqueue);
-//ncptl_finalize();
-
- /* Finalization code specific to the c_mpi backend */
-CODES_MPI_Finalize();
-mpi_is_running = 0;
-exitcode = 0;
-
- /* Return an exit status code. */
-return exitcode;
-}
-
-/*************************************************************************/
-/***************************** MAIN ROUTINE ******************************/
-/*************************************************************************/
-
-/* Program execution starts here. */
-static int hotpotato_main (int argc, char *argv[])
-{
- /* Declare variables needed by all C-based backends. */
-CONC_EVENT * eventlist;   /* List of events to execute */
-ncptl_int numevents;   /* Number of entries in eventlist[] */
-
- /* ----- Initialization ----- */
-conc_initialize (argc, argv);
-eventlist = (CONC_EVENT *) ncptl_queue_contents (eventqueue, 0);
-numevents = ncptl_queue_length (eventqueue);
-sendrequests = (MPI_Request *) ncptl_queue_contents (sendreqQ, 0);
-sendstatuses = (MPI_Status *) ncptl_queue_contents (sendstatQ, 0);
-recvrequests = (MPI_Request *) ncptl_queue_contents (recvreqQ, 0);
-recvstatuses = (MPI_Status *) ncptl_queue_contents (recvstatQ, 0);
-starttime = ncptl_time();
-
- /* ----- Event-list processing ----- */
-conc_process_events (eventlist, 0, numevents-1, 1);
-
- /* ----- Finalization ----- */
-return conc_finalize();
-}
-
-/* fill in function pointers for this method */
-struct codes_conceptual_bench hotpotato_bench = 
-{
-.program_name = "hotpotato",
-.conceptual_main = hotpotato_main,
-};

From 807f83eece9627bc20d5c1cb7ba4385570c8f4c7 Mon Sep 17 00:00:00 2001
From: Xin Wang <xwang149@hawk.iit.edu>
Date: Tue, 22 Jan 2019 15:14:51 -0600
Subject: [PATCH 13/15] Delete conc-latency.c

---
 .../conceputal-skeleton-apps/conc-latency.c   | 1269 -----------------
 1 file changed, 1269 deletions(-)
 delete mode 100644 src/workload/conceputal-skeleton-apps/conc-latency.c

diff --git a/src/workload/conceputal-skeleton-apps/conc-latency.c b/src/workload/conceputal-skeleton-apps/conc-latency.c
deleted file mode 100644
index 2751a4f3..00000000
--- a/src/workload/conceputal-skeleton-apps/conc-latency.c
+++ /dev/null
@@ -1,1269 +0,0 @@
-/**********************************************************************
- * This file was generated by coNCePTuaL on Mon Jan  7 23:09:59 2019
- * using the c_mpi backend (C + MPI).
- * Do not modify this file; modify /Users/xin/macworkspace/codes-dev/codes/scripts/conceptual_benchmarks/latency.ncptl instead.
- *
- * Entire source program
- * ---------------------
- *   # A ping-pong latency test written in coNCePTuaL
- *   
- *   Require language version "1.5".
- *   
- *   # Parse the command line.
- *   reps is "Number of repetitions of each message size" and comes from
- *    "--reps" or "-r" with default 1000.
- *   maxbytes is "Maximum number of bytes to transmit" and comes from
- *    "--maxbytes" or "-m" with default 1M.
- *   
- *   # Ensure that we have a peer with whom to communicate.
- *   Assert that "the latency test requires at least two tasks" with num_tasks>=2.
- *   
- *   # Perform the benchmark.
- *   For each msgsize in {0}, {1, 2, 4, ..., maxbytes} {
- *     for reps repetitions {
- *       task 0 resets its counters then
- *       task 0 sends a msgsize byte message to task 1 then
- *       task 1 sends a msgsize byte message to task 0 then
- *       task 0 logs the msgsize as "Bytes" and
- *                   the median of elapsed_usecs/2 as "1/2 RTT (usecs)"
- *     } then
- *     task 0 computes aggregates
- *   }
- **********************************************************************/
-
-#include "codes/codes-conc-addon.h"
-/*****************
- * Include files *
- *****************/
-
-/* Header files needed by all C-based backends */
-#include <stdio.h>
-#include <string.h>
-#include <ncptl/ncptl.h>
-
-/* Header files specific to the c_mpi backend */
-#include <mpi.h>
-#include <stdarg.h>
-
-/**********
- * Macros *
- **********/
-
-/* Define the maximum loop trip count that we're willing to unroll fully. */
-#define CONC_MAX_UNROLL 5
-
-/* Specify the minimum number of trial iterations in each FOR <time> loop. */
-#define CONC_FOR_TIME_TRIALS 1
-
-/* Define a macro that rounds a double to a ncptl_int. */
-#define CONC_DBL2INT(D) ((ncptl_int)((D)+0.5))
-
-/* Define a macro that increments a buffer pointer by a byte offset. */
-#define CONC_GETBUFPTR(S) ((void *)((char *)thisev->s.S.buffer + thisev->s.S.bufferofs))
-
-/* Implement ncptl_func_task_of in terms of ncptl_physical_to_virtual. */
-#define ncptl_func_task_of(P) ((ncptl_int)(P) < 0LL || (ncptl_int)(P) >= var_num_tasks ? -1LL : ncptl_physical_to_virtual (procmap, (ncptl_int)(P)))
-#define ncptl_dfunc_task_of(P) ((double) ncptl_func_task_of(P))
-
-/* Implement ncptl_func_processor_of in terms of ncptl_virtual_to_physical. */
-#define ncptl_func_processor_of(V) ((ncptl_int)(V) < 0LL || (ncptl_int)(V) >= var_num_tasks ? -1LL : ncptl_virtual_to_physical (procmap, (ncptl_int)(V)))
-#define ncptl_dfunc_processor_of(V) ((double) ncptl_func_processor_of(V))
-
-/* Define a macro that increments REDUCE's alternate buffer pointer by a byte offset. */
-#define CONC_GETALTBUFPTR(S) ((void *)((char *)thisev->s.S.altbuffer + thisev->s.S.bufferofs))
-
-/* Estimate the number of unique communicators that this program will need.
- * (The tradeoff is one of initialization time versus memory consumption.) */
-#define ESTIMATED_COMMUNICATORS 128
-
-/* Specify an operation to use for all reduction operations. */
-#define REDUCE_OPERATION MPI_SUM
-#define REDUCE_OPERATION_NAME "MPI_SUM"
-
-
-/*********************
- * Type declarations *
- *********************/
-
-/* Enumerate the various mechanisms used to implement MULTICAST statements. */
-typedef enum {
-CONC_MCAST_MPI_BCAST,       /* One to many */
-CONC_MCAST_MPI_ALLTOALL,    /* Many to many, same data to all */ 
-CONC_MCAST_MPI_ALLTOALLV,   /* General many to many */
-CONC_MCAST_MPI_NUM_FUNCS    /* Number of the above */
-} CONC_MCAST_MPI_FUNC;
-
-/* Define the type of event to perform. */
-typedef enum {
-EV_SEND,     /* Synchronous send */
-EV_ASEND,    /* Asynchronous send */
-EV_RECV,     /* Synchronous receive */
-EV_ARECV,    /* Asynchronous receive */
-EV_WAIT,     /* Wait for all asynchronous sends/receives to complete */
-EV_DELAY,    /* Spin or sleep */
-EV_TOUCH,    /* Touch a region of memory */
-EV_SYNC,     /* Barrier synchronization */
-EV_RESET,    /* Reset counters */
-EV_STORE,    /* Store all counters' current values */
-EV_RESTORE,  /* Restore the previously pushed counter values */
-EV_FLUSH,    /* Compute aggregate functions for log-file columns */
-EV_MCAST,    /* Synchronous multicast */
-EV_REDUCE,   /* Reduction with or without a subsequent multicast */
-EV_BTIME,    /* Beginning of a timed loop */
-EV_ETIME,    /* Ending of a timed loop */
-EV_REPEAT,   /* Repeatedly process the next N events */
-EV_SUPPRESS, /* Suppress writing to the log and standard output */
-EV_NEWSTMT,  /* Beginning of a new top-level statement */
-EV_CODE,     /* None of the above */
-NUM_EVS      /* Number of event types in CONC_EVENT_TYPE */
-} CONC_EVENT_TYPE;
-
-/* Describe a synchronous or asynchronous send event. */
-typedef struct {
-ncptl_int dest;         /* Destination task */
-ncptl_int size;         /* Number of bytes to send */
-ncptl_int alignment;    /* Message alignment (in bytes) */
-ncptl_int pendingsends; /* # of outstanding sends */
-ncptl_int pendingrecvs; /* # of outstanding receives */
-ncptl_int buffernum;    /* Buffer # to send from */
-ncptl_int bufferofs;    /* Byte offset into the message buffer */
-ncptl_int tag;          /* Tag to use for selective receives */
-int misaligned;         /* 1=misaligned from a page; 0=align as specified */
-int touching;           /* 1=touch every word before sending */
-int verification;       /* 1=fill message buffer with known contents */
-void *buffer;           /* Pointer to message memory */
-MPI_Request * handle;   /* MPI handle representing an asynchronous send */
-} CONC_SEND_EVENT;
-
-/* Describe a synchronous or asynchronous receive event. */
-typedef struct {
-ncptl_int source;       /* Source task */
-ncptl_int size;         /* Number of bytes to receive */
-ncptl_int alignment;    /* Message alignment (in bytes) */
-ncptl_int pendingsends; /* # of outstanding sends */
-ncptl_int pendingrecvs; /* # of outstanding receives */
-ncptl_int buffernum;    /* Buffer # to receive into */
-ncptl_int bufferofs;    /* Byte offset into the message buffer */
-ncptl_int tag;          /* Tag to use for selective receives */
-int misaligned;         /* 1=misaligned from a page; 0=align as specified */
-int touching;           /* 1=touch every word after reception */
-int verification;       /* 1=verify that all bits are correct */
-void *buffer;           /* Pointer to message memory */
-MPI_Request * handle;   /* MPI handle representing an asynchronous receive */
-} CONC_RECV_EVENT;
-
-/* Describe a wait-for-asynchronous-completions event. */
-typedef struct {
-ncptl_int numsends;     /* # of sends we expect to complete. */
-ncptl_int numrecvs;     /* # of receives we expect to complete. */
-ncptl_int numrecvbytes; /* # of bytes we expect to receive-complete */
-ncptl_int *touchedlist;    /* List of receives that need to be touched */
-ncptl_int numtouches;         /* # of elements in the above */
-} CONC_WAIT_EVENT;
-
-/* Describe a spin or sleep delay. */
-typedef struct {
-uint64_t microseconds;  /* Length of delay in microseconds */
-int spin0sleep1;        /* 0=spin; 1=sleep */
-} CONC_DELAY_EVENT;
-
-/* Describe a barrier synchronization event. */
-typedef struct {
-MPI_Comm communicator;   /* Set of tasks to synchronize */
-} CONC_SYNC_EVENT;
-
-/* Describe a walk over a memory-region. */
-typedef struct {
-ncptl_int regionbytes;  /* Size in bytes of the region to touch */
-ncptl_int bytestride;   /* Stride in bytes to touch */
-ncptl_int numaccesses;  /* Number of words to touch */
-ncptl_int wordsize;     /* Size in bytes of each touch */
-ncptl_int firstbyte;    /* Byte offset of the first byte to touch */
-} CONC_TOUCH_EVENT;
-
-/* Describe a synchronous multicast event. */
-typedef struct {
-ncptl_int source;       /* Source task, -1 in the many-to-many case */
-ncptl_int size;         /* Number of bytes to send */
-ncptl_int alignment;    /* Message alignment (in bytes) */
-ncptl_int pendingsends; /* # of outstanding sends */
-ncptl_int pendingrecvs; /* # of outstanding receives */
-ncptl_int buffernum;    /* Buffer # to send/receive from */
-ncptl_int bufferofs;    /* Byte offset into the message buffer */
-ncptl_int tag;          /* Tag to use for selective receives */
-int misaligned;         /* 1=misaligned from a page; 0=align as specified */
-int touching;           /* 1=touch every word before sending */
-int verification;       /* 1=verify that all bits are correct */
-void *buffer;           /* Pointer to message memory */
-ncptl_int size2;   /* Number of bytes to receive in the many-to-many case */
-ncptl_int bufferofs2;   /* Byte offset into the message buffer in the many-to-many case */
-void * buffer2;   /* Pointer to receive-message memory in the many-to-many case */
-MPI_Comm communicator;   /* Set of tasks to multicast to/from */
-int root;   /* source's rank within communicator */
-int * sndvol;   /* Volume of data to send to each rank in the communicator */
-int * snddisp;   /* Offset from buffer of each message to send */
-int * rcvvol;   /* Volume of data to receive from each rank in the communicator */
-int * rcvdisp;   /* Offset from buffer2 of each message to receive */
-CONC_MCAST_MPI_FUNC mpi_func;   /* MPI function to use to perform the multicast */
-} CONC_MCAST_EVENT;
-
-/* Describe a reduction event. */
-typedef struct {
-ncptl_int numitems;     /* # of items to reduce */
-ncptl_int itemsize;     /* # of bytes per item */
-ncptl_int alignment;    /* Message alignment (in bytes) */
-ncptl_int pendingsends; /* # of outstanding sends */
-ncptl_int pendingrecvs; /* # of outstanding receives */
-ncptl_int buffernum;    /* Buffer # to send/receive from */
-ncptl_int bufferofs;    /* Byte offset into the message buffer */
-ncptl_int tag;          /* Tag to use for selective receives */
-int misaligned;         /* 1=misaligned from a page; 0=align as specified */
-int touching;           /* 1=touch every word before sending/after receiving */
-int sending;            /* 1=we're a sender */
-int receiving;          /* 1=we're a receiver */
-void *buffer;           /* Pointer to message memory */
-void * altbuffer;   /* Pointer to additional message memory */
-MPI_Comm sendcomm;   /* Set of tasks to reduce from */
-MPI_Comm recvcomm;   /* Set of tasks to reduce to */
-MPI_Datatype datatype;   /* MPI datatype to reduce */
-int reducetype;   /* 0=reduce; 1=allreduce; 2=reduce+bcast */
-ncptl_int reduceroot;   /* Root task of the reduction if reducetype is 0 or 2 */
-ncptl_int bcastroot;   /* Root task of the multicast if reducetype is 2 */
-} CONC_REDUCE_EVENT;
-
-/* Describe an event representing the beginning of a timed loop. */
-typedef struct {
-uint64_t usecs;         /* Requested loop duration */
-uint64_t warmup_usecs;  /* Requested duration of warmup loops */
-uint64_t starttime;     /* Time at which the loop state last changed */
-uint64_t itersleft;     /* # of iterations remaining */
-uint64_t previters;     /* # of iterations we performed last time */
-int prev_quiet;         /* Previous value of suppress_output */
-int timing_trial;       /* 1=performing a timing trial; 0=running for real */
-volatile int finished;  /* 1=time has expired; 0=still ticking */
-} CONC_BTIME_EVENT;
-
-/* Describe an event representing the end of a timed loop. */
-typedef struct {
-ncptl_int begin_event;  /* Index into eventlist[] of the corresponding BTIME event */
-} CONC_ETIME_EVENT;
-
-/* Describe an event representing repetitions of subsequent events. */
-typedef struct {
-ncptl_int end_event;    /* Index into eventlist[] of the last event to repeat */
-ncptl_int numreps;      /* # of repetitions to perform */
-} CONC_REPEAT_EVENT;
-
-/* Describe an event representing output suppression (either on or off). */
-typedef struct conc_suppress_event {
-int quiet;              /* 0=allow output; 1=suppress it */
-int prev_quiet;         /* Previous value of suppress_output */
-ncptl_int matching_event;  /* Event ID of the "suppression on" event */
-uint64_t stop_elapsed_usecs;   /* Time at which we suppressed output */
-} CONC_SUPPRESS_EVENT;
-
-/* Describe an event representing arbitrary code to execute at run time. */
-typedef struct {
-ncptl_int number;       /* Unique number corresponding to a specific piece of code */NCPTL_VIRT_PHYS_MAP *procmap;  /* Current mapping between tasks and processors */
-ncptl_int var_msgsize;   /* Copy of var_msgsize to use within a piece of code */
-} CONC_CODE_EVENT;
-
-/* Describe an arbitrary coNCePTuaL event. */
-typedef struct {
-CONC_EVENT_TYPE type;          /* Type of event */
-union {
-CONC_SEND_EVENT send;          /* Send state */
-CONC_RECV_EVENT recv;          /* Receive state */
-CONC_WAIT_EVENT wait;          /* Wait-for-completions state */
-CONC_DELAY_EVENT delay;        /* State for spins and sleeps */
-CONC_TOUCH_EVENT touch;        /* State for memory touching */
-CONC_SYNC_EVENT sync;          /* Synchronization state */
-CONC_MCAST_EVENT mcast;        /* Multicast state */
-CONC_REDUCE_EVENT reduce;      /* Reduction state */
-CONC_BTIME_EVENT btime;        /* Timed-loop state */
-CONC_ETIME_EVENT etime;        /* Additional timed-loop state */
-CONC_REPEAT_EVENT rep;         /* Repeated-events state */
-CONC_SUPPRESS_EVENT suppress;  /* State for suppressing output */
-CONC_CODE_EVENT code;          /* State for arbitrary code */
-} s;
-} CONC_EVENT;
-
-/* Fully specify an arbitrary for() loop (used by FOR EACH). */
-typedef struct {
-NCPTL_QUEUE *list_comp;  /* NULL=ordinary list; other=list comprehension values */
-int integral;        /* 1=integral values; 0=floating-point values */
-enum {               /* Comparison of loop variable to end variable */
-CONC_LEQ,                /* Increasing progression */
-CONC_GEQ                 /* Decreasing progression */
-} comparator;
-enum {               /* How to increment the loop variable */
-CONC_ADD,                /* Arithmetically */
-CONC_MULT,               /* Geometrically increasing */
-CONC_DIV                 /* Geometrically decreasing */
-} increment;
-union {
-struct {
-ncptl_int loopvar;   /* Loop variable */
-ncptl_int prev_loopvar; /* Previous value of loop variable */
-ncptl_int startval;  /* Initial value of loop variable */
-ncptl_int endval;    /* Value not to exceed */
-ncptl_int incval;    /* Loop-variable increment */
-} i;
-struct {
-double loopvar;      /* Loop variable */
-double prev_loopvar; /* Previous value of loop variable */
-double startval;     /* Initial value of loop variable */
-double endval;       /* Value not to exceed */
-double incval;       /* Loop-variable increment */
-} d;
-} u;
-} LOOPBOUNDS;
-
-/********************
- * Global variables *
- ********************/
-
-/* Variables exported to coNCePTuaL programs */
-static ncptl_int var_bytes_received = 0;   /* Total number of bytes received */
-static ncptl_int var_msgs_received = 0;   /* Total number of messages received */
-static ncptl_int var_bit_errors = 0;   /* Total number of bit errors observed */
-static ncptl_int var_total_msgs = 0;   /* Sum of messages sent and messages received */
-static ncptl_int var_msgs_sent = 0;   /* Total number of messages sent */
-static ncptl_int var_bytes_sent = 0;   /* Total number of bytes sent */
-static ncptl_int var_num_tasks = 1;   /* Number of tasks running the program */
-static ncptl_int var_elapsed_usecs = 0;   /* Elapsed time in microseconds */
-static ncptl_int var_total_bytes = 0;   /* Sum of bytes sent and bytes received */
-
-/* Dummy variable to help mark other variables as used */
-static union {
-ncptl_int ni;
-int i;
-void *vp;
-} conc_dummy_var;
-
-/* Variables used internally by boilerplate code */
-static uint64_t starttime;   /* Time the clock was last reset (microseconds) */
-static ncptl_int pendingrecvs = 0;   /* Current # of outstanding receives */
-static ncptl_int pendingrecvbytes = 0; /* Current # of bytes in outstanding receives */
-static NCPTL_QUEUE *touchedqueue;      /* Queue of asynchronous receives to touch */
-static ncptl_int pendingsends = 0;   /* Current # of outstanding sends */
-static NCPTL_QUEUE *eventqueue;   /* List of coNCePTuaL events to perform */
-static int within_time_loop = 0;   /* 1=we're within a FOR <time> loop */
-static int suppress_output = 0;    /* 1=suppress output to stdout and the log file */
-static void *touch_region = NULL;   /* Memory region to touch */
-static ncptl_int touch_region_size = 0;   /* # of bytes in the above */
-static int virtrank;    /* This task's virtual rank in the computation */
-static int physrank;    /* This task's physical rank in the computation */
-static NCPTL_VIRT_PHYS_MAP *procmap;  /* Virtual to physical rank mapping */
-static NCPTL_LOG_FILE_STATE *logstate;   /* Opaque object representing all log-file state */
-static char *logfile_uuid;   /* Execution UUID to write to every log file */
-static char *logfiletmpl;   /* Template for the log file's name */
-//static char *logfiletmpl_default;   /* Default value of the above */
-
-/* Global variables specific to the c_mpi backend */
-static ncptl_int mpi_is_running = 0;   /* 1=MPI has been initialized */
-static NCPTL_QUEUE * recvreqQ;   /* List of MPI receive requests */
-static MPI_Request * recvrequests;   /* List version of recvreqQ */
-static NCPTL_QUEUE * recvstatQ;   /* List of MPI receive statuses */
-static MPI_Status * recvstatuses;   /* List version of recvstatQ */
-static NCPTL_QUEUE * sendreqQ;   /* List of MPI send requests */
-static MPI_Request * sendrequests;   /* List version of sendreqQ */
-static NCPTL_QUEUE * sendstatQ;   /* List of MPI send statuses */
-static MPI_Status * sendstatuses;   /* List version of sendstatQ */
-static NCPTL_SET * communicators;   /* Map from an array of processor flags to an MPI communicator */
-static MPI_Errhandler mpi_error_handler;   /* Handle to handle_MPI_error() */
-static ncptl_int mpi_tag_ub;   /* Upper bound on an MPI tag value */
-static ncptl_int conc_mcast_tallies[CONC_MCAST_MPI_NUM_FUNCS] = {0};   /* Tallies of (static) multicast implementation functions */
-
-/* Program-specific variables */
-static ncptl_int var_reps;   /* Number of repetitions of each message size (command-line argument) */
-static ncptl_int var_maxbytes;   /* Maximum number of bytes to transmit (command-line argument) */
-
-/*************************
- * Function declarations *
- *************************/
-
-/* Make MPI errors invoke ncptl_fatal(). */
-static void handle_MPI_error (MPI_Comm *comm, int *errcode, ...)
-{
-va_list args;
-char errstring[MPI_MAX_ERROR_STRING];
-int errstrlen;
-
-va_start (args, errcode);
-if (MPI_Error_string (*errcode, errstring, &errstrlen) == MPI_SUCCESS)
-ncptl_fatal ("MPI run-time error: %s", errstring);
-else
-ncptl_fatal ("MPI aborted with unrecognized error code %d", *errcode);
-conc_dummy_var.vp = (void *) comm;   /* Prevent the compiler from complaining that comm is unused. */
-va_end (args);
-}
-
-/* Perform the equivalent of CODES_MPI_Comm_rank() for an arbitrary process. */
-static int rank_in_MPI_communicator (MPI_Comm subcomm, int global_rank)
-{
-  MPI_Group world_group;   /* Group associated with MPI_COMM_WORLD */
-  MPI_Group subgroup;      /* Group associate with subcomm */
-  int subrank;             /* global_rank's rank within subcomm */
-
-  MPI_Comm_group (MPI_COMM_WORLD, &world_group);
-  MPI_Comm_group (subcomm, &subgroup);
-  MPI_Group_translate_ranks (world_group, 1, &global_rank, subgroup, &subrank);
-  return subrank;
-}
-
-/* Map an arbitrary tag to within MPI's valid range of [0, mpi_tag_ub]. */
-static ncptl_int map_tag_into_MPI_range (ncptl_int tag)
-{
-if (tag == NCPTL_INT_MIN)
- /* Avoid taking the absolute value of NCPTL_INT_MIN. */
-tag = 555666773LL;   /* Arbitrary value */
-tag = ncptl_func_abs (tag);   /* Only nonnegatives values are allowed. */
-if (mpi_tag_ub < NCPTL_INT_MAX)
-tag %= mpi_tag_ub + 1;
-return tag;
-}
-
-/* Given an array of task in/out booleans return an MPI
- * communicator that represents the "in" tasks. */
-static MPI_Comm define_MPI_communicator (char *procflags)
-{
-MPI_Comm *existing_comm;    /* Previously defined MPI communicator */
-MPI_Comm new_comm;          /* Newly defined MPI communicator */
-
-existing_comm = (MPI_Comm *) ncptl_set_find (communicators, (void *)procflags);
-if (existing_comm)
-return *existing_comm;
-(void) MPI_Comm_split (MPI_COMM_WORLD, (int)procflags[physrank], physrank, &new_comm);
-//(void) MPI_Errhandler_set (new_comm, mpi_error_handler);
-ncptl_set_insert (communicators, (void *)procflags, (void *)&new_comm);
-return define_MPI_communicator (procflags);
-}
-
-/* Return 1 if a sequence loop will take at least one trip. */
-static int conc_seq_nonempty (LOOPBOUNDS *seq)
-{
-ncptl_int startval;   /* Integer version of seq's startval element */
-ncptl_int endval;   /* Integer version of seq's endval element */
-
-if (seq->integral) {
-startval = seq->u.i.startval;
-endval = seq->u.i.endval;
-}
-else {
-startval = CONC_DBL2INT (seq->u.d.startval);
-endval = CONC_DBL2INT (seq->u.d.endval);
-}
-switch (seq->comparator) {
-case CONC_LEQ:
-return startval <= endval;
-
-case CONC_GEQ:
-return startval >= endval;
-
-default:
-ncptl_fatal ("Internal error -- unknown comparator");
-}
-return -1;     /* Appease idiotic compilers. */
-}
-
-/* Initialize a sequence loop. */
-static void conc_seq_init (LOOPBOUNDS *seq)
-{
-if (seq->integral) {
-seq->u.i.loopvar = seq->u.i.startval;
-seq->u.i.prev_loopvar = seq->u.i.loopvar - 1;
-}
-else {
-seq->u.d.loopvar = seq->u.d.startval;
-seq->u.d.prev_loopvar = seq->u.d.loopvar - 1.0;
-}
-}
-
-/* Return 1 if a sequence loop should continue, 0 when finished. */
-static int conc_seq_continue (LOOPBOUNDS *seq)
-{
-LOOPBOUNDS seq_int;   /* Integer equivalent of *seq */
-if (seq->integral)
-seq_int = *seq;
-else {
-seq_int.u.i.loopvar = CONC_DBL2INT (seq->u.d.loopvar);
-seq_int.u.i.prev_loopvar = CONC_DBL2INT (seq->u.d.prev_loopvar);
-seq_int.u.i.endval = CONC_DBL2INT (seq->u.d.endval);
-}
-
-if (seq_int.u.i.loopvar == seq_int.u.i.prev_loopvar)
-return 0;
-switch (seq->comparator) {
-case CONC_LEQ:
-return seq_int.u.i.loopvar <= seq_int.u.i.endval;
-
-case CONC_GEQ:
-return seq_int.u.i.loopvar >= seq_int.u.i.endval;
-
-default:
-ncptl_fatal ("Internal error -- unknown comparator");
-}
-return -1;     /* Appease idiotic compilers. */
-}
-
-/* Proceed to the next iteration of a sequence loop. */
-static void conc_seq_next (LOOPBOUNDS *seq)
-{
-if (seq->integral) {
-seq->u.i.prev_loopvar = seq->u.i.loopvar;
-switch (seq->increment) {
-case CONC_ADD:
-seq->u.i.loopvar += seq->u.i.incval;
-break;
-
-case CONC_MULT:
-seq->u.i.loopvar *= seq->u.i.incval;
-break;
-
-case CONC_DIV:
-seq->u.i.loopvar /= seq->u.i.incval;
-break;
-
-default:
-ncptl_fatal ("Internal error -- unknown incrementer");
-}
-}
-else {
-seq->u.d.prev_loopvar = seq->u.d.loopvar;
-switch (seq->increment) {
-case CONC_ADD:
-seq->u.d.loopvar += seq->u.d.incval;
-break;
-
-case CONC_MULT:
-seq->u.d.loopvar *= seq->u.d.incval;
-break;
-
-case CONC_DIV:
-seq->u.d.loopvar /= seq->u.d.incval;
-break;
-
-default:
-ncptl_fatal ("Internal error -- unknown incrementer");
-}
-}
-}
-
-/* Inhibit the compiler from complaining that
- * certain variables are defined but not used.
- * This function should never be called. */
-static void conc_mark_variables_used (void)
-{
-conc_dummy_var.ni = var_bytes_received;
-conc_dummy_var.ni = var_msgs_received;
-conc_dummy_var.ni = var_bit_errors;
-conc_dummy_var.ni = var_total_msgs;
-conc_dummy_var.ni = var_msgs_sent;
-conc_dummy_var.ni = var_bytes_sent;
-conc_dummy_var.ni = var_num_tasks;
-conc_dummy_var.ni = var_elapsed_usecs;
-conc_dummy_var.ni = var_total_bytes;
-conc_dummy_var.ni = pendingrecvbytes;
-conc_dummy_var.ni = touch_region_size;
-conc_dummy_var.vp = touch_region;
-conc_dummy_var.i  = within_time_loop;
-conc_dummy_var.i  = suppress_output;
-rank_in_MPI_communicator (MPI_COMM_WORLD, 0);
-}
-
-/* Allocate a new event of a given type and return a pointer to it. */
-static CONC_EVENT *conc_allocate_event (CONC_EVENT_TYPE type)
-{
-CONC_EVENT *newevent = (CONC_EVENT *) ncptl_queue_allocate (eventqueue);
-
-newevent->type = type;
-return newevent;
-}
-
-/* Declare an exit handler that gets called automatically when the
- * program terminates, whether successfully or not. */
-static void conc_exit_handler (void)
-{
-if (mpi_is_running)
-MPI_Abort (MPI_COMM_WORLD, 1);
-}
-
-
-/* Initialize coNCePTuaL, the messaging layer, and this program itself. */
-static void conc_initialize (int argc, char *argv[])
-{
- /* Variables needed by all C-based backends */
-CONC_EVENT * eventlist;   /* List of events to execute */
-ncptl_int numevents;   /* Number of entries in eventlist[] */
-int help_only = 0;   /* 1=User specified --help; save time by skipping ncptl_init() */
-char * argv0 = strrchr(argv[0], '/') ? strrchr(argv[0], '/')+1 : argv[0];   /* Base name of the executable program */
-int i;   /* Generic loop variable */
-
- /* Declare all of our command-line arguments. */
-NCPTL_CMDLINE arguments[] = {
-{ NCPTL_TYPE_STRING, NULL, "logfile", 'L', "Log-file template", {0}},
-{ NCPTL_TYPE_INT, NULL, "reps", 'r', "Number of repetitions of each message size", {0}},
-{ NCPTL_TYPE_INT, NULL, "maxbytes", 'm', "Maximum number of bytes to transmit", {0}}
-};
-
- /* Incorporate the complete coNCePTuaL source code as an array
-//  * for use by ncptl_log_write_prologue(). */
-char *sourcecode[] = {
-"# A ping-pong latency test written in coNCePTuaL",
-"",
-"Require language version \"1.5\".",
-"",
-"# Parse the command line.",
-"reps is \"Number of repetitions of each message size\" and comes from",
-" \"--reps\" or \"-r\" with default 1000.",
-"maxbytes is \"Maximum number of bytes to transmit\" and comes from",
-" \"--maxbytes\" or \"-m\" with default 1M.",
-"",
-"# Ensure that we have a peer with whom to communicate.",
-"Assert that \"the latency test requires at least two tasks\" with num_tasks>=2.",
-"",
-"# Perform the benchmark.",
-"For each msgsize in {0}, {1, 2, 4, ..., maxbytes} {",
-"  for reps repetitions {",
-"    task 0 resets its counters then",
-"    task 0 sends a msgsize byte message to task 1 then",
-"    task 1 sends a msgsize byte message to task 0 then",
-"    task 0 logs the msgsize as \"Bytes\" and",
-"                the median of elapsed_usecs/2 as \"1/2 RTT (usecs)\"",
-"  } then",
-"  task 0 computes aggregates",
-"}",
-NULL
-};
-
- /* Variables specific to the c_mpi backend */
-int num_tasks;   /* int version of var_num_tasks needed by CODES_MPI_Comm_size() */
-char * procflags;   /* Array of 1s representing an all-task MPI communicator */
-MPI_Comm comm_world = MPI_COMM_WORLD;   /* Copy of MPI_COMM_WORLD that we can take the address of */
-void * attr_val;   /* Pointed to the value of MPI_TAG_UB */
-int attr_flag = 0;   /* true=MPI_TAG_UB was extracted; false=not extracted */
-char log_key_str[128];   /* String representing the range of valid MPI tags */
-
- /* As a special case, if the command line contains --help, then skip
-  * the coNCePTuaL initialization step. */
-for (i=1; i<argc; i++)
-if (!strcmp(argv[i], "--"))
-break;
-else
-if (!strcmp(argv[i], "--help") || !strcmp(argv[i], "-?")) {
-argv[1] = "-?";   /* Guaranteed to work, even with getopt() */
-help_only = 1;
-break;
-}
-
- /* Perform various initializations specific to the c_mpi backend. */
- /* Initialize MPI. */
-//(void) MPI_Init(&argc, &argv);
-mpi_is_running = 1;
-
- /* Initialize the coNCePTuaL run-time library. */
-if (!help_only)
-//ncptl_init (NCPTL_RUN_TIME_VERSION, argv[0]);
-//(void) atexit (conc_exit_handler);
-
- /* Initialize the communication routines needed by the c_mpi backend. */
-//(void) MPI_Errhandler_create ((MPI_Handler_function *)handle_MPI_error, &mpi_error_handler);
-//(void) MPI_Errhandler_set (MPI_COMM_WORLD, mpi_error_handler);
-(void) CODES_MPI_Comm_rank(MPI_COMM_WORLD, &physrank);
-(void) CODES_MPI_Comm_size(MPI_COMM_WORLD, &num_tasks);
-var_num_tasks = (ncptl_int) num_tasks;
-//(void) MPI_Comm_get_attr(MPI_COMM_WORLD, MPI_TAG_UB, &attr_val, &attr_flag);
-mpi_tag_ub = (ncptl_int) (attr_flag ? *(int *)attr_val : 32767);
-
- /* Generate and broadcast a UUID. */
-//logfile_uuid = ncptl_log_generate_uuid();
-//(void) CODES_MPI_Bcast ((void *)logfile_uuid, 37, MPI_CHAR, 0, MPI_COMM_WORLD);
-
- /* Plug variables and default values into the NCPTL_CMDLINE structure. */
-arguments[0].variable = (CMDLINE_VALUE *) &logfiletmpl;
-arguments[1].variable = (CMDLINE_VALUE *) &var_reps;
-arguments[1].defaultvalue.intval = 1000LL;
-arguments[2].variable = (CMDLINE_VALUE *) &var_maxbytes;
-arguments[2].defaultvalue.intval = 1048576LL;
-//logfiletmpl_default = (char *) ncptl_malloc (strlen(argv0) + 15, 0);
-//sprintf (logfiletmpl_default, "%s-%%p.log", argv0);
-//arguments[0].defaultvalue.stringval = logfiletmpl_default;
-
- /* Parse the command line. */
-mpi_is_running = 0;   /* Don't invoke MPI_Abort() after --help. */
-ncptl_parse_command_line (argc, argv, arguments, sizeof(arguments)/sizeof(NCPTL_CMDLINE));
-if (help_only)
-ncptl_fatal ("Internal error in the c_generic backend: failed to exit after giving help");
-mpi_is_running = 1;
-
- /* Establish a mapping from (virtual) task IDs to (physical) ranks. */
-procmap = ncptl_allocate_task_map (var_num_tasks);
-virtrank = ncptl_physical_to_virtual (procmap, physrank);
-
- /* Perform initializations specific to the c_mpi backend. */
-//ncptl_log_add_comment ("MPI send routines", "MPI_Send() and CODES_MPI_Isend()");
-//ncptl_log_add_comment ("MPI reduction operation", REDUCE_OPERATION_NAME);
-//sprintf (log_key_str, "[0, %" NICS "]", mpi_tag_ub);
-//ncptl_log_add_comment ("MPI tag range", log_key_str);
-//
-// /* Open the log file and write some standard prologue information to it. */
-//logstate = ncptl_log_open (logfiletmpl, physrank);
-////ncptl_log_write_prologue (logstate, argv[0], logfile_uuid, "c_mpi", "C + MPI",
-//var_num_tasks,
-//arguments, sizeof(arguments)/sizeof(NCPTL_CMDLINE),
-//sourcecode);
-//ncptl_free (logfile_uuid);
-
- /* Allocate a variety of dynamically growing queues. */
-eventqueue = ncptl_queue_init (sizeof (CONC_EVENT));
-touchedqueue = ncptl_queue_init (sizeof (ncptl_int));
-
- /* Perform initializations specific to the c_mpi backend. */
-sendreqQ = ncptl_queue_init (sizeof (MPI_Request));
-sendstatQ = ncptl_queue_init (sizeof (MPI_Status));
-recvreqQ = ncptl_queue_init (sizeof (MPI_Request));
-recvstatQ = ncptl_queue_init (sizeof (MPI_Status));
-communicators = ncptl_set_init (ESTIMATED_COMMUNICATORS, var_num_tasks*sizeof(char), sizeof(MPI_Comm));procflags = (char *) ncptl_malloc (var_num_tasks*sizeof(char), 0);
-for (i=0; i<var_num_tasks; i++)
-procflags[i] = 1;
-ncptl_set_insert (communicators, (void *)procflags, (void *)&comm_world);
-ncptl_free (procflags);
-
- /****************************************************
-  * Generated, program-specific initialization code. *
-  ****************************************************/
-
- /* ASSERT THAT the latency test requires at least two tasks. */
-if (!((var_num_tasks)>=(2LL)))
-ncptl_fatal ("Assertion failure: the latency test requires at least two tasks");
- /* FOR EACH var_msgsize IN [(['0LL'], None), (['1LL', '2LL', '4LL'], 'var_maxbytes')]... */
-{
-LOOPBOUNDS loopbounds[2];   /* List of range descriptions */
-ncptl_int rangenum;   /* Current offset into loopbounds[] */
-ncptl_int initial_vals[3];   /* Cache of the initial, enumerated values */
-ncptl_int final_val;   /* Cache of the final value */
-
- /* Write range 0's loop bounds, "next" function, and termination function to loopbounds[0]. */
-loopbounds[0].list_comp = NULL;
-initial_vals[0] = 0LL;
-final_val = 0LL;
-loopbounds[0].integral = 1;
-loopbounds[0].u.i.startval = initial_vals[0];
-loopbounds[0].u.i.endval = final_val;
-loopbounds[0].comparator = CONC_LEQ;
-loopbounds[0].increment = CONC_ADD;
-loopbounds[0].u.i.incval = 1LL;
-
- /* Write range 1's loop bounds, "next" function, and termination function to loopbounds[1]. */
-loopbounds[1].list_comp = NULL;
-initial_vals[0] = 1LL;
-initial_vals[1] = 2LL;
-initial_vals[2] = 4LL;
-final_val = var_maxbytes;
-loopbounds[1].integral = 1;
-loopbounds[1].u.i.startval = initial_vals[0];
-loopbounds[1].u.i.endval = final_val;
-if (initial_vals[1]-initial_vals[0]==initial_vals[2]-initial_vals[1]) {
- /* Arithmetic progression */
-loopbounds[1].comparator = initial_vals[0]<=initial_vals[1] ? CONC_LEQ : CONC_GEQ;
-loopbounds[1].increment = CONC_ADD;
-loopbounds[1].u.i.incval = initial_vals[1]-initial_vals[0];
-if (!loopbounds[1].u.i.incval)
-loopbounds[1].u.i.incval = 1LL;   /* Handle {x,x,x,...,x} case (constant value) */
-}
-else
-if (initial_vals[0] && initial_vals[1] && initial_vals[0]<initial_vals[1] && initial_vals[0]*(initial_vals[1]/initial_vals[0])==initial_vals[1] && initial_vals[1]*(initial_vals[2]/initial_vals[1])==initial_vals[2] && initial_vals[1]/initial_vals[0]==initial_vals[2]/initial_vals[1]) {
- /* Geometric progression (increasing, integral multiplier) */
-loopbounds[1].comparator = CONC_LEQ;
-loopbounds[1].increment = CONC_MULT;
-loopbounds[1].u.i.incval = initial_vals[1]/initial_vals[0];
-}
-else
-if (initial_vals[1] && initial_vals[2] && initial_vals[0]>initial_vals[1] && initial_vals[1]*(initial_vals[0]/initial_vals[1])==initial_vals[0] && initial_vals[2]*(initial_vals[1]/initial_vals[2])==initial_vals[1] && initial_vals[0]/initial_vals[1]==initial_vals[1]/initial_vals[2]) {
- /* Geometric progression (decreasing, integral multiplier) */
-loopbounds[1].comparator = CONC_GEQ;
-loopbounds[1].increment = CONC_DIV;
-loopbounds[1].u.i.incval = initial_vals[0]/initial_vals[1];
-}
-else
-if (initial_vals[0] && initial_vals[1]) {
-double initial_vals_d[3];   /* Cache of the initial, enumerated values, but in floating-point context */
-double avg_factor;   /* Average multiplier for terms in the sequence */
-initial_vals_d[0] = 1.0;
-initial_vals_d[1] = 2.0;
-initial_vals_d[2] = 4.0;
-avg_factor = (initial_vals_d[1]/initial_vals_d[0] + initial_vals_d[2]/initial_vals_d[1]) / 2.0;
-if (CONC_DBL2INT(initial_vals_d[0]*avg_factor)==initial_vals[1] && CONC_DBL2INT(initial_vals_d[1]*avg_factor)==initial_vals[2]) {
- /* Geometric progression (decreasing or non-integral multiplier) */
-loopbounds[1].comparator = initial_vals[0]<initial_vals[1] ? CONC_LEQ : CONC_GEQ;
-loopbounds[1].integral = 0;
-loopbounds[1].increment = CONC_MULT;
-loopbounds[1].u.d.startval = initial_vals_d[0];
-loopbounds[1].u.d.endval = (double)var_maxbytes;
-loopbounds[1].u.d.incval = avg_factor;
-}
-else
-ncptl_fatal ("Unable to find an arithmetic or geometric pattern to {%" NICS ", %" NICS ", %" NICS ", ..., %" NICS "}", initial_vals[0], initial_vals[1], initial_vals[2], final_val);
-}
-else
-ncptl_fatal ("Unable to find an arithmetic or geometric pattern to {%" NICS ", %" NICS ", %" NICS ", ..., %" NICS "}", initial_vals[0], initial_vals[1], initial_vals[2], final_val);
-
- /* Now that we've defined all of our ranges we iterate over each range
-  * and each element within each range. */
-for (rangenum=0LL; rangenum<2; rangenum++) {
-LOOPBOUNDS * thisrange = &loopbounds[rangenum];   /* Current range */
-if (conc_seq_nonempty (thisrange))
-for (conc_seq_init (thisrange);
-conc_seq_continue (thisrange);
-conc_seq_next (thisrange)) {
-ncptl_int var_msgsize;
-if (thisrange->list_comp == NULL)
- /* The loopvar field contains the value for var_msgsize. */
-var_msgsize = thisrange->integral ? thisrange->u.i.loopvar : CONC_DBL2INT(thisrange->u.d.loopvar);
-else
- /* The queue of list-comprehension values provides the value for var_msgsize. */
-var_msgsize = *(ncptl_int *)ncptl_queue_pop(thisrange->list_comp);
-{
- /* FOR var_reps REPETITIONS... */
-{
-ncptl_int numreps = var_reps;   /* Total # of repetitions to perform */
-int unroll_loop = numreps <= CONC_MAX_UNROLL;   /* 1=unroll loop; 0=use a REPEAT event */
-ncptl_int repnum;   /* Current repetition number */
-CONC_EVENT * repevent = NULL;   /* Event designating repetition */
-ncptl_int repeventnum = -1LL;   /* Event number corresponding to repevent */
- /* Conditionally unroll the loop. */
-for (repnum=0; repnum<(unroll_loop?numreps:1); repnum++) {
- /* Allocate a repeat event if we have more than one repetition. */
-if (!unroll_loop && numreps > 1LL) {
-repeventnum = ncptl_queue_length (eventqueue);
-repevent =  conc_allocate_event (EV_REPEAT);
-repevent->s.rep.numreps = numreps;
-}
-
- /* Output a loop body if we have at least one repetition. */
-if (unroll_loop || numreps > 0LL) {
- /* TASK 0LL RESET THEIR COUNTERS */
-if ((0LL) == virtrank) {   /* TASK 0LL */
- /* The current coNCePTuaL statement applies to our task. */
-(void) conc_allocate_event (EV_RESET);
-}
- /* THEN... */
- /* TASK 0LL RECEIVES FROM TASK 1LL */
-if ((0LL)>=0 && (0LL)<var_num_tasks) {
- /* 0LL now represents one of the tasks that will send to us. */
-ncptl_int ivar_a_task = 0LL;
-if (virtrank == (1LL)) {
- /* In this scope, we must be a message recipient. */
- /* Prepare to receive 1LL messages. */
-ncptl_int numreps = 1LL;   /* Number of messages */
-if (numreps > 1LL) {
-CONC_EVENT * repeatev;   /* Event specifying the number of repetitions to perform */
-repeatev = conc_allocate_event (EV_REPEAT);
-repeatev->s.rep.end_event = ncptl_queue_length(eventqueue);
-repeatev->s.rep.numreps = numreps;
-}
-
- /* Ensure we have at least one message to receive. */
-if (numreps > 0LL) {
-CONC_EVENT *thisev = conc_allocate_event (EV_RECV);
-
- /* Fill in all of the fields of a receive-event structure. */
-thisev->s.recv.source = ncptl_virtual_to_physical (procmap, ivar_a_task);
-thisev->s.recv.size = var_msgsize;
-thisev->s.recv.alignment = 0LL;
-thisev->s.recv.misaligned = 0;
-thisev->s.recv.touching = 0;
-thisev->s.recv.verification = 0;
-thisev->s.recv.tag = 0LL;
-thisev->s.recv.pendingsends = pendingsends;
-thisev->s.recv.pendingrecvs = pendingrecvs;
-thisev->s.recv.bufferofs = 0LL;
-thisev->s.recv.buffernum = thisev->s.recv.verification ? pendingsends+pendingrecvs : 0;
-(void) ncptl_malloc_message (thisev->s.recv.size+thisev->s.recv.bufferofs, thisev->s.recv.alignment, thisev->s.recv.buffernum, thisev->s.recv.misaligned);
-thisev->s.recv.buffer = NULL;
-thisev->s.recv.tag = map_tag_into_MPI_range (thisev->s.recv.tag);
-}
-}
-}
- /* TASK 0LL SENDS TO TASK 1LL */
-if ((0LL) == virtrank) {   /* TASK 0LL */
- /* The current coNCePTuaL statement applies to our task. */
-ncptl_int virtdest = 1LL;
-if (virtdest>=0 && virtdest<var_num_tasks) {
- /* In this scope, 1LL represents a single receiver. */
- /* Prepare to send 1LL messages. */
-ncptl_int numreps = 1LL;   /* Number of messages */
-if (numreps > 1LL) {
-CONC_EVENT * repeatev;   /* Event specifying the number of repetitions to perform */
-repeatev = conc_allocate_event (EV_REPEAT);
-repeatev->s.rep.end_event = ncptl_queue_length(eventqueue);
-repeatev->s.rep.numreps = numreps;
-}
-
- /* Ensure we have at least one message to send. */
-if (numreps > 0LL) {
-CONC_EVENT *thisev = conc_allocate_event (EV_SEND);
-if (virtrank == (1LL))
-ncptl_fatal ("Send-to-self deadlock encountered on task %d in line 18 of the source code", virtrank);
-
- /* Fill in all of the fields of a send-event structure. */
-thisev->s.send.dest = ncptl_virtual_to_physical (procmap, 1LL);
-thisev->s.send.size = var_msgsize;
-thisev->s.send.alignment = 0LL;
-thisev->s.send.misaligned = 0;
-thisev->s.send.touching = 0;
-thisev->s.send.verification = 0;
-thisev->s.send.tag = 0LL;
-thisev->s.send.pendingsends = pendingsends;
-thisev->s.send.pendingrecvs = pendingrecvs;
-thisev->s.send.bufferofs = 0LL;
-thisev->s.send.buffernum = thisev->s.send.verification ? pendingsends+pendingrecvs : 0;
-(void) ncptl_malloc_message (thisev->s.send.size+thisev->s.send.bufferofs, thisev->s.send.alignment, thisev->s.send.buffernum, thisev->s.send.misaligned);
-thisev->s.send.buffer = NULL;
-thisev->s.send.tag = map_tag_into_MPI_range (thisev->s.send.tag);
-}
-}
-}
- /* THEN... */
- /* TASK 1LL RECEIVES FROM TASK 0LL */
-if ((1LL)>=0 && (1LL)<var_num_tasks) {
- /* 1LL now represents one of the tasks that will send to us. */
-ncptl_int ivar_b_task = 1LL;
-if (virtrank == (0LL)) {
- /* In this scope, we must be a message recipient. */
- /* Prepare to receive 1LL messages. */
-ncptl_int numreps = 1LL;   /* Number of messages */
-if (numreps > 1LL) {
-CONC_EVENT * repeatev;   /* Event specifying the number of repetitions to perform */
-repeatev = conc_allocate_event (EV_REPEAT);
-repeatev->s.rep.end_event = ncptl_queue_length(eventqueue);
-repeatev->s.rep.numreps = numreps;
-}
-
- /* Ensure we have at least one message to receive. */
-if (numreps > 0LL) {
-CONC_EVENT *thisev = conc_allocate_event (EV_RECV);
-
- /* Fill in all of the fields of a receive-event structure. */
-thisev->s.recv.source = ncptl_virtual_to_physical (procmap, ivar_b_task);
-thisev->s.recv.size = var_msgsize;
-thisev->s.recv.alignment = 0LL;
-thisev->s.recv.misaligned = 0;
-thisev->s.recv.touching = 0;
-thisev->s.recv.verification = 0;
-thisev->s.recv.tag = 0LL;
-thisev->s.recv.pendingsends = pendingsends;
-thisev->s.recv.pendingrecvs = pendingrecvs;
-thisev->s.recv.bufferofs = 0LL;
-thisev->s.recv.buffernum = thisev->s.recv.verification ? pendingsends+pendingrecvs : 0;
-(void) ncptl_malloc_message (thisev->s.recv.size+thisev->s.recv.bufferofs, thisev->s.recv.alignment, thisev->s.recv.buffernum, thisev->s.recv.misaligned);
-thisev->s.recv.buffer = NULL;
-thisev->s.recv.tag = map_tag_into_MPI_range (thisev->s.recv.tag);
-}
-}
-}
- /* TASK 1LL SENDS TO TASK 0LL */
-if ((1LL) == virtrank) {   /* TASK 1LL */
- /* The current coNCePTuaL statement applies to our task. */
-ncptl_int virtdest = 0LL;
-if (virtdest>=0 && virtdest<var_num_tasks) {
- /* In this scope, 0LL represents a single receiver. */
- /* Prepare to send 1LL messages. */
-ncptl_int numreps = 1LL;   /* Number of messages */
-if (numreps > 1LL) {
-CONC_EVENT * repeatev;   /* Event specifying the number of repetitions to perform */
-repeatev = conc_allocate_event (EV_REPEAT);
-repeatev->s.rep.end_event = ncptl_queue_length(eventqueue);
-repeatev->s.rep.numreps = numreps;
-}
-
- /* Ensure we have at least one message to send. */
-if (numreps > 0LL) {
-CONC_EVENT *thisev = conc_allocate_event (EV_SEND);
-if (virtrank == (0LL))
-ncptl_fatal ("Send-to-self deadlock encountered on task %d in line 19 of the source code", virtrank);
-
- /* Fill in all of the fields of a send-event structure. */
-thisev->s.send.dest = ncptl_virtual_to_physical (procmap, 0LL);
-thisev->s.send.size = var_msgsize;
-thisev->s.send.alignment = 0LL;
-thisev->s.send.misaligned = 0;
-thisev->s.send.touching = 0;
-thisev->s.send.verification = 0;
-thisev->s.send.tag = 0LL;
-thisev->s.send.pendingsends = pendingsends;
-thisev->s.send.pendingrecvs = pendingrecvs;
-thisev->s.send.bufferofs = 0LL;
-thisev->s.send.buffernum = thisev->s.send.verification ? pendingsends+pendingrecvs : 0;
-(void) ncptl_malloc_message (thisev->s.send.size+thisev->s.send.bufferofs, thisev->s.send.alignment, thisev->s.send.buffernum, thisev->s.send.misaligned);
-thisev->s.send.buffer = NULL;
-thisev->s.send.tag = map_tag_into_MPI_range (thisev->s.send.tag);
-}
-}
-}
- /* THEN... */
- /* TASK 0LL LOGS "Bytes" AND "1/2 RTT (usecs)" */
-if ((0LL) == virtrank) {   /* TASK 0LL */
- /* The current coNCePTuaL statement applies to our task. */
-CONC_EVENT *thisev = conc_allocate_event (EV_CODE);
-thisev->s.code.number = 0;
-thisev->s.code.procmap = NULL;
-thisev->s.code.var_msgsize = var_msgsize;
-}
-}
-}
-
- /* Assign the number of events to repeat, now that we know that number. */
-if (!unroll_loop && numreps > 1LL) {
-repevent = repeventnum + (CONC_EVENT *) ncptl_queue_contents (eventqueue, 0);
-repevent->s.rep.end_event = ncptl_queue_length (eventqueue) - 1;
-}
-}
- /* THEN... */
- /* TASK 0LL COMPUTES AGGREGATES */
-if ((0LL) == virtrank) {   /* TASK 0LL */
- /* The current coNCePTuaL statement applies to our task. */
-(void) conc_allocate_event (EV_FLUSH);
-}
-}
-}
-}
-}
- /* Begin a new top-level statement. */
-(void) conc_allocate_event (EV_NEWSTMT);
-
- /*************************
-  * More boilerplate code *
-  *************************/
-
- /* Abort if the program will terminate with pending messages. */
-if (pendingsends && pendingrecvs)
-ncptl_fatal("Neglected to await the completion of %" NICS " asynchronous %s and %" NICS " asynchronous %s",
-pendingsends, pendingsends==1LL ? "send" : "sends",
-pendingrecvs, pendingrecvs==1LL ? "receive" : "receives");
-else
-if (pendingsends)
-ncptl_fatal("Neglected to await the completion of %" NICS " asynchronous %s",
-pendingsends, pendingsends==1LL ? "send" : "sends");
-else
-if (pendingrecvs)
-ncptl_fatal("Neglected to await the completion of %" NICS " asynchronous %s",
-pendingrecvs, pendingrecvs==1LL ? "receive" : "receives");
-
- /* Allocate memory for non-unique messages and asynchronous
-  * message handles now that we know how much memory we need
-  * to allocate. */
-eventlist = (CONC_EVENT *) ncptl_queue_contents (eventqueue, 0);
-numevents = ncptl_queue_length (eventqueue);
-sendrequests = (MPI_Request *) ncptl_queue_contents (sendreqQ, 0);
-recvrequests = (MPI_Request *) ncptl_queue_contents (recvreqQ, 0);
-for (i=0; i<numevents; i++) {
-CONC_EVENT *thisev = &eventlist[i];   /* Cache of the current event */
-switch (thisev->type) {
-case EV_SEND:
-if (!thisev->s.send.buffer)
-thisev->s.send.buffer = ncptl_malloc_message (thisev->s.send.bufferofs + thisev->s.send.size,
-thisev->s.send.alignment,
-thisev->s.send.buffernum,
-thisev->s.send.misaligned);
-if (thisev->s.send.verification)
-ncptl_fill_buffer (CONC_GETBUFPTR(send), thisev->s.send.bufferofs + thisev->s.send.size, -1);
-break;
-
-case EV_RECV:
-if (!thisev->s.recv.buffer)
-thisev->s.recv.buffer = ncptl_malloc_message (thisev->s.recv.bufferofs + thisev->s.recv.size,
-thisev->s.recv.alignment,
-thisev->s.recv.buffernum,
-thisev->s.recv.misaligned);
-if (thisev->s.recv.verification)
-ncptl_fill_buffer (CONC_GETBUFPTR(recv), thisev->s.recv.bufferofs + thisev->s.recv.size, -1);
-break;
-
-default:
-break;
-}
-}
-}
-
-/* Process a subset of the events in a given event list. */
-static void conc_process_events (CONC_EVENT *eventlist,
-ncptl_int firstev, ncptl_int lastev, ncptl_int numreps)
-{
-CONC_EVENT * thisev;   /* Cache of the current event */
-CONC_EVENT * thisev_first = &eventlist[firstev];   /* Cache of the first event */
-ncptl_int i;   /* Iterate over events. */
-ncptl_int j;   /* Iterate over repetitions. */
-
- /* Declarations specific to the c_mpi backend */
-MPI_Status status;   /* Not needed but required by CODES_MPI_Recv() */
-
- /* Process from event firstev to event lastev (both inclusive). */
-for (j=numreps; j>0; j--)
-for (i=firstev, thisev=thisev_first; i<=lastev; i++, thisev++) {
- /* Declare variables needed by all C-based backends. */
-
- /* Process a single event. */
-switch (thisev->type) {
-case EV_SEND:
- /* Synchronous send */
-(void) CODES_MPI_Send (CONC_GETBUFPTR(send),
-(int)thisev->s.send.size, MPI_BYTE,
-(int)thisev->s.send.dest, (int)thisev->s.send.tag, MPI_COMM_WORLD);
-break;
-
-case EV_RECV:
- /* Synchronous receive */
-(void) CODES_MPI_Recv (CONC_GETBUFPTR(recv),
-(int)thisev->s.recv.size, MPI_BYTE,
-(int)thisev->s.recv.source, (int)thisev->s.recv.tag,
-MPI_COMM_WORLD, &status);
-break;
-
-case EV_RESET:
- /* Reset all of the counters exported to coNCePTuaL programs. */
-starttime = ncptl_time();
-break;
-
-case EV_FLUSH:
- /* Force all aggregate functions to produce a result. */
-if (!suppress_output) {
-uint64_t stop_elapsed_usecs = ncptl_time();
-var_elapsed_usecs = stop_elapsed_usecs - starttime;
-//ncptl_log_compute_aggregates (logstate);
-starttime += ncptl_time() - stop_elapsed_usecs;
-}
-break;
-
-case EV_REPEAT:
- /* Repeatedly perform the next batch of events. */
-conc_process_events (eventlist, i+1, thisev->s.rep.end_event, thisev->s.rep.numreps);
-i = thisev->s.rep.end_event;
-thisev = &eventlist[i];
-break;
-
-case EV_NEWSTMT:
- /* Begin a new table in the log file. */
-if (!suppress_output) {
-uint64_t stop_elapsed_usecs = ncptl_time();
-var_elapsed_usecs = stop_elapsed_usecs - starttime;
-//ncptl_log_commit_data (logstate);
-starttime += ncptl_time() - stop_elapsed_usecs;
-}
-break;
-
-case EV_CODE:
- /* Execute an arbitrary piece of code. */
-switch (thisev->s.code.number) {
-case 0:
- /* TASK 0LL LOGS "Bytes" AND "1/2 RTT (usecs)" */
-if (!suppress_output) {
-uint64_t stop_elapsed_usecs = ncptl_time();
-var_elapsed_usecs = stop_elapsed_usecs - starttime;
-//ncptl_log_write (logstate, 0, "Bytes", NCPTL_FUNC_ONLY, 0.0, (double)thisev->s.code.var_msgsize);
-//ncptl_log_write (logstate, 1, "1/2 RTT (usecs)", NCPTL_FUNC_MEDIAN, 0.0, ((double)var_elapsed_usecs)/(2.0));
-starttime += ncptl_time() - stop_elapsed_usecs;
-}
-break;
-
-default:
- /* The C code generation module must be broken. */
-ncptl_fatal ("Internal error: unknown EV_CODE block %" NICS, thisev->s.code.number);
-break;
-}
-break;
-
-default:
- /* The c_generic backend or the c_mpi backend must be broken. */
-ncptl_fatal ("Internal error: unknown event type %d", thisev->type);
-break;
-}
-}
-}
-
-/* Finish up cleanly and return a status code. */
-static int conc_finalize (void)
-{
-int exitcode = 0;   /* Program exit code (to pass to exit()) */
-
- /* Declarations specific to the c_mpi backend */
-//int mpiresult;   /* Return code from MPI_Finalize() */
-//char log_key_str[128];   /* String representing the range of valid MPI tags */
-//
-// /* Finalization code specific to the c_mpi backend */
-//log_key_str[0] = '\0';
-//if (conc_mcast_tallies[CONC_MCAST_MPI_BCAST] > 0) {
-//char onefuncstr[50];
-//sprintf (onefuncstr, "%sCODES_MPI_Bcast()*%" NICS,
-//log_key_str[0] == '\0' ? "" : " ", conc_mcast_tallies[CONC_MCAST_MPI_BCAST]);
-//strcat (log_key_str, onefuncstr);
-//}
-//if (conc_mcast_tallies[CONC_MCAST_MPI_ALLTOALL] > 0) {
-//char onefuncstr[50];
-//sprintf (onefuncstr, "%sCODES_MPI_Alltoall()*%" NICS,
-//log_key_str[0] == '\0' ? "" : " ", conc_mcast_tallies[CONC_MCAST_MPI_ALLTOALL]);
-//strcat (log_key_str, onefuncstr);
-//}
-//if (conc_mcast_tallies[CONC_MCAST_MPI_ALLTOALLV] > 0) {
-//char onefuncstr[50];
-//sprintf (onefuncstr, "%sCODES_MPI_Alltoallv()*%" NICS,
-//log_key_str[0] == '\0' ? "" : " ", conc_mcast_tallies[CONC_MCAST_MPI_ALLTOALLV]);
-//strcat (log_key_str, onefuncstr);
-//}
-//if (log_key_str[0] != '\0')
-//ncptl_log_add_comment ("Multicast functions used (statically)", log_key_str);
-//
-// /* Write a standard epilogue to the log file. */
-////ncptl_log_commit_data (logstate);
-////ncptl_log_write_epilogue (logstate);
-//ncptl_log_close (logstate);
-
- /* Inform the run-time library that it's no longer needed. */
-//ncptl_queue_empty (eventqueue);
-//ncptl_free (eventqueue);
-//ncptl_finalize();
-
- /* Finalization code specific to the c_mpi backend */
-CODES_MPI_Finalize();
-mpi_is_running = 0;
-exitcode = 0;
-
- /* Return an exit status code. */
-return exitcode;
-}
-
-/*************************************************************************/
-/***************************** MAIN ROUTINE ******************************/
-/*************************************************************************/
-
-/* Program execution starts here. */
-static int latency_main (int argc, char *argv[])
-{
- /* Declare variables needed by all C-based backends. */
-CONC_EVENT * eventlist;   /* List of events to execute */
-ncptl_int numevents;   /* Number of entries in eventlist[] */
-
- /* ----- Initialization ----- */
-conc_initialize (argc, argv);
-eventlist = (CONC_EVENT *) ncptl_queue_contents (eventqueue, 0);
-numevents = ncptl_queue_length (eventqueue);
-sendrequests = (MPI_Request *) ncptl_queue_contents (sendreqQ, 0);
-sendstatuses = (MPI_Status *) ncptl_queue_contents (sendstatQ, 0);
-recvrequests = (MPI_Request *) ncptl_queue_contents (recvreqQ, 0);
-recvstatuses = (MPI_Status *) ncptl_queue_contents (recvstatQ, 0);
-starttime = ncptl_time();
-
- /* ----- Event-list processing ----- */
-conc_process_events (eventlist, 0, numevents-1, 1);
-
- /* ----- Finalization ----- */
-return conc_finalize();
-}
-
-/* fill in function pointers for this method */
-struct codes_conceptual_bench latency_bench = 
-{
-.program_name = "latency",
-.conceptual_main = latency_main,
-};

From 3219d20008b515e964fd31154d498d3eecf8d29f Mon Sep 17 00:00:00 2001
From: Xin Wang <xwang149@hawk.iit.edu>
Date: Tue, 22 Jan 2019 15:15:08 -0600
Subject: [PATCH 14/15] Delete conc-latencyall.c

---
 .../conc-latencyall.c                         | 1292 -----------------
 1 file changed, 1292 deletions(-)
 delete mode 100644 src/workload/conceputal-skeleton-apps/conc-latencyall.c

diff --git a/src/workload/conceputal-skeleton-apps/conc-latencyall.c b/src/workload/conceputal-skeleton-apps/conc-latencyall.c
deleted file mode 100644
index eedfe10b..00000000
--- a/src/workload/conceputal-skeleton-apps/conc-latencyall.c
+++ /dev/null
@@ -1,1292 +0,0 @@
-/**********************************************************************
- * This file was generated by coNCePTuaL on Mon Jan  7 23:10:00 2019
- * using the c_mpi backend (C + MPI).
- * Do not modify this file; modify /Users/xin/macworkspace/codes-dev/codes/scripts/conceptual_benchmarks/latency-all.ncptl instead.
- *
- * Entire source program
- * ---------------------
- *   # An all-pairs ping-pong latency test written in coNCePTuaL
- *   # By Scott Pakin <pakin@lanl.gov>
- *   
- *   Require language version "1.5".
- *   
- *   # Parse the command line.
- *   reps is "Number of repetitions of each message size" and comes from "--reps" or "-r" with default 1000.
- *   maxbytes is "Maximum number of bytes to transmit" and comes from "--maxbytes" or "-m" with default 1M.
- *   
- *   # Ensure that we have a peer with whom to communicate.
- *   Assert that "the latency test requires at least two tasks" with num_tasks>=2.
- *   
- *   # Perform the benchmark.
- *   For each msgsize in {0}, {1, 2, 4, ..., maxbytes} {
- *     for reps repetitions {
- *       tasks ev such that ev is even reset their counters then
- *       tasks ev such that ev is even send a msgsize byte message to task ev+1 then
- *       tasks od such that od is odd send a msgsize byte message to task od-1 then
- *       tasks ev such that ev is even log the msgsize as "Bytes" and
- *                                         the median of elapsed_usecs/2 as "1/2 RTT (usecs)"
- *     } then
- *     tasks ev such that ev is even compute aggregates
- *   }
- **********************************************************************/
-
-#include "codes/codes-conc-addon.h"
-/*****************
- * Include files *
- *****************/
-
-/* Header files needed by all C-based backends */
-#include <stdio.h>
-#include <string.h>
-#include <ncptl/ncptl.h>
-
-/* Header files specific to the c_mpi backend */
-#include <mpi.h>
-#include <stdarg.h>
-
-/**********
- * Macros *
- **********/
-
-/* Define the maximum loop trip count that we're willing to unroll fully. */
-#define CONC_MAX_UNROLL 5
-
-/* Specify the minimum number of trial iterations in each FOR <time> loop. */
-#define CONC_FOR_TIME_TRIALS 1
-
-/* Define a macro that rounds a double to a ncptl_int. */
-#define CONC_DBL2INT(D) ((ncptl_int)((D)+0.5))
-
-/* Define a macro that increments a buffer pointer by a byte offset. */
-#define CONC_GETBUFPTR(S) ((void *)((char *)thisev->s.S.buffer + thisev->s.S.bufferofs))
-
-/* Implement ncptl_func_task_of in terms of ncptl_physical_to_virtual. */
-#define ncptl_func_task_of(P) ((ncptl_int)(P) < 0LL || (ncptl_int)(P) >= var_num_tasks ? -1LL : ncptl_physical_to_virtual (procmap, (ncptl_int)(P)))
-#define ncptl_dfunc_task_of(P) ((double) ncptl_func_task_of(P))
-
-/* Implement ncptl_func_processor_of in terms of ncptl_virtual_to_physical. */
-#define ncptl_func_processor_of(V) ((ncptl_int)(V) < 0LL || (ncptl_int)(V) >= var_num_tasks ? -1LL : ncptl_virtual_to_physical (procmap, (ncptl_int)(V)))
-#define ncptl_dfunc_processor_of(V) ((double) ncptl_func_processor_of(V))
-
-/* Define a macro that increments REDUCE's alternate buffer pointer by a byte offset. */
-#define CONC_GETALTBUFPTR(S) ((void *)((char *)thisev->s.S.altbuffer + thisev->s.S.bufferofs))
-
-/* Estimate the number of unique communicators that this program will need.
- * (The tradeoff is one of initialization time versus memory consumption.) */
-#define ESTIMATED_COMMUNICATORS 128
-
-/* Specify an operation to use for all reduction operations. */
-#define REDUCE_OPERATION MPI_SUM
-#define REDUCE_OPERATION_NAME "MPI_SUM"
-
-
-/*********************
- * Type declarations *
- *********************/
-
-/* Enumerate the various mechanisms used to implement MULTICAST statements. */
-typedef enum {
-CONC_MCAST_MPI_BCAST,       /* One to many */
-CONC_MCAST_MPI_ALLTOALL,    /* Many to many, same data to all */ 
-CONC_MCAST_MPI_ALLTOALLV,   /* General many to many */
-CONC_MCAST_MPI_NUM_FUNCS    /* Number of the above */
-} CONC_MCAST_MPI_FUNC;
-
-/* Define the type of event to perform. */
-typedef enum {
-EV_SEND,     /* Synchronous send */
-EV_ASEND,    /* Asynchronous send */
-EV_RECV,     /* Synchronous receive */
-EV_ARECV,    /* Asynchronous receive */
-EV_WAIT,     /* Wait for all asynchronous sends/receives to complete */
-EV_DELAY,    /* Spin or sleep */
-EV_TOUCH,    /* Touch a region of memory */
-EV_SYNC,     /* Barrier synchronization */
-EV_RESET,    /* Reset counters */
-EV_STORE,    /* Store all counters' current values */
-EV_RESTORE,  /* Restore the previously pushed counter values */
-EV_FLUSH,    /* Compute aggregate functions for log-file columns */
-EV_MCAST,    /* Synchronous multicast */
-EV_REDUCE,   /* Reduction with or without a subsequent multicast */
-EV_BTIME,    /* Beginning of a timed loop */
-EV_ETIME,    /* Ending of a timed loop */
-EV_REPEAT,   /* Repeatedly process the next N events */
-EV_SUPPRESS, /* Suppress writing to the log and standard output */
-EV_NEWSTMT,  /* Beginning of a new top-level statement */
-EV_CODE,     /* None of the above */
-NUM_EVS      /* Number of event types in CONC_EVENT_TYPE */
-} CONC_EVENT_TYPE;
-
-/* Describe a synchronous or asynchronous send event. */
-typedef struct {
-ncptl_int dest;         /* Destination task */
-ncptl_int size;         /* Number of bytes to send */
-ncptl_int alignment;    /* Message alignment (in bytes) */
-ncptl_int pendingsends; /* # of outstanding sends */
-ncptl_int pendingrecvs; /* # of outstanding receives */
-ncptl_int buffernum;    /* Buffer # to send from */
-ncptl_int bufferofs;    /* Byte offset into the message buffer */
-ncptl_int tag;          /* Tag to use for selective receives */
-int misaligned;         /* 1=misaligned from a page; 0=align as specified */
-int touching;           /* 1=touch every word before sending */
-int verification;       /* 1=fill message buffer with known contents */
-void *buffer;           /* Pointer to message memory */
-MPI_Request * handle;   /* MPI handle representing an asynchronous send */
-} CONC_SEND_EVENT;
-
-/* Describe a synchronous or asynchronous receive event. */
-typedef struct {
-ncptl_int source;       /* Source task */
-ncptl_int size;         /* Number of bytes to receive */
-ncptl_int alignment;    /* Message alignment (in bytes) */
-ncptl_int pendingsends; /* # of outstanding sends */
-ncptl_int pendingrecvs; /* # of outstanding receives */
-ncptl_int buffernum;    /* Buffer # to receive into */
-ncptl_int bufferofs;    /* Byte offset into the message buffer */
-ncptl_int tag;          /* Tag to use for selective receives */
-int misaligned;         /* 1=misaligned from a page; 0=align as specified */
-int touching;           /* 1=touch every word after reception */
-int verification;       /* 1=verify that all bits are correct */
-void *buffer;           /* Pointer to message memory */
-MPI_Request * handle;   /* MPI handle representing an asynchronous receive */
-} CONC_RECV_EVENT;
-
-/* Describe a wait-for-asynchronous-completions event. */
-typedef struct {
-ncptl_int numsends;     /* # of sends we expect to complete. */
-ncptl_int numrecvs;     /* # of receives we expect to complete. */
-ncptl_int numrecvbytes; /* # of bytes we expect to receive-complete */
-ncptl_int *touchedlist;    /* List of receives that need to be touched */
-ncptl_int numtouches;         /* # of elements in the above */
-} CONC_WAIT_EVENT;
-
-/* Describe a spin or sleep delay. */
-typedef struct {
-uint64_t microseconds;  /* Length of delay in microseconds */
-int spin0sleep1;        /* 0=spin; 1=sleep */
-} CONC_DELAY_EVENT;
-
-/* Describe a barrier synchronization event. */
-typedef struct {
-MPI_Comm communicator;   /* Set of tasks to synchronize */
-} CONC_SYNC_EVENT;
-
-/* Describe a walk over a memory-region. */
-typedef struct {
-ncptl_int regionbytes;  /* Size in bytes of the region to touch */
-ncptl_int bytestride;   /* Stride in bytes to touch */
-ncptl_int numaccesses;  /* Number of words to touch */
-ncptl_int wordsize;     /* Size in bytes of each touch */
-ncptl_int firstbyte;    /* Byte offset of the first byte to touch */
-} CONC_TOUCH_EVENT;
-
-/* Describe a synchronous multicast event. */
-typedef struct {
-ncptl_int source;       /* Source task, -1 in the many-to-many case */
-ncptl_int size;         /* Number of bytes to send */
-ncptl_int alignment;    /* Message alignment (in bytes) */
-ncptl_int pendingsends; /* # of outstanding sends */
-ncptl_int pendingrecvs; /* # of outstanding receives */
-ncptl_int buffernum;    /* Buffer # to send/receive from */
-ncptl_int bufferofs;    /* Byte offset into the message buffer */
-ncptl_int tag;          /* Tag to use for selective receives */
-int misaligned;         /* 1=misaligned from a page; 0=align as specified */
-int touching;           /* 1=touch every word before sending */
-int verification;       /* 1=verify that all bits are correct */
-void *buffer;           /* Pointer to message memory */
-ncptl_int size2;   /* Number of bytes to receive in the many-to-many case */
-ncptl_int bufferofs2;   /* Byte offset into the message buffer in the many-to-many case */
-void * buffer2;   /* Pointer to receive-message memory in the many-to-many case */
-MPI_Comm communicator;   /* Set of tasks to multicast to/from */
-int root;   /* source's rank within communicator */
-int * sndvol;   /* Volume of data to send to each rank in the communicator */
-int * snddisp;   /* Offset from buffer of each message to send */
-int * rcvvol;   /* Volume of data to receive from each rank in the communicator */
-int * rcvdisp;   /* Offset from buffer2 of each message to receive */
-CONC_MCAST_MPI_FUNC mpi_func;   /* MPI function to use to perform the multicast */
-} CONC_MCAST_EVENT;
-
-/* Describe a reduction event. */
-typedef struct {
-ncptl_int numitems;     /* # of items to reduce */
-ncptl_int itemsize;     /* # of bytes per item */
-ncptl_int alignment;    /* Message alignment (in bytes) */
-ncptl_int pendingsends; /* # of outstanding sends */
-ncptl_int pendingrecvs; /* # of outstanding receives */
-ncptl_int buffernum;    /* Buffer # to send/receive from */
-ncptl_int bufferofs;    /* Byte offset into the message buffer */
-ncptl_int tag;          /* Tag to use for selective receives */
-int misaligned;         /* 1=misaligned from a page; 0=align as specified */
-int touching;           /* 1=touch every word before sending/after receiving */
-int sending;            /* 1=we're a sender */
-int receiving;          /* 1=we're a receiver */
-void *buffer;           /* Pointer to message memory */
-void * altbuffer;   /* Pointer to additional message memory */
-MPI_Comm sendcomm;   /* Set of tasks to reduce from */
-MPI_Comm recvcomm;   /* Set of tasks to reduce to */
-MPI_Datatype datatype;   /* MPI datatype to reduce */
-int reducetype;   /* 0=reduce; 1=allreduce; 2=reduce+bcast */
-ncptl_int reduceroot;   /* Root task of the reduction if reducetype is 0 or 2 */
-ncptl_int bcastroot;   /* Root task of the multicast if reducetype is 2 */
-} CONC_REDUCE_EVENT;
-
-/* Describe an event representing the beginning of a timed loop. */
-typedef struct {
-uint64_t usecs;         /* Requested loop duration */
-uint64_t warmup_usecs;  /* Requested duration of warmup loops */
-uint64_t starttime;     /* Time at which the loop state last changed */
-uint64_t itersleft;     /* # of iterations remaining */
-uint64_t previters;     /* # of iterations we performed last time */
-int prev_quiet;         /* Previous value of suppress_output */
-int timing_trial;       /* 1=performing a timing trial; 0=running for real */
-volatile int finished;  /* 1=time has expired; 0=still ticking */
-} CONC_BTIME_EVENT;
-
-/* Describe an event representing the end of a timed loop. */
-typedef struct {
-ncptl_int begin_event;  /* Index into eventlist[] of the corresponding BTIME event */
-} CONC_ETIME_EVENT;
-
-/* Describe an event representing repetitions of subsequent events. */
-typedef struct {
-ncptl_int end_event;    /* Index into eventlist[] of the last event to repeat */
-ncptl_int numreps;      /* # of repetitions to perform */
-} CONC_REPEAT_EVENT;
-
-/* Describe an event representing output suppression (either on or off). */
-typedef struct conc_suppress_event {
-int quiet;              /* 0=allow output; 1=suppress it */
-int prev_quiet;         /* Previous value of suppress_output */
-ncptl_int matching_event;  /* Event ID of the "suppression on" event */
-uint64_t stop_elapsed_usecs;   /* Time at which we suppressed output */
-} CONC_SUPPRESS_EVENT;
-
-/* Describe an event representing arbitrary code to execute at run time. */
-typedef struct {
-ncptl_int number;       /* Unique number corresponding to a specific piece of code */NCPTL_VIRT_PHYS_MAP *procmap;  /* Current mapping between tasks and processors */
-ncptl_int var_msgsize;   /* Copy of var_msgsize to use within a piece of code */
-} CONC_CODE_EVENT;
-
-/* Describe an arbitrary coNCePTuaL event. */
-typedef struct {
-CONC_EVENT_TYPE type;          /* Type of event */
-union {
-CONC_SEND_EVENT send;          /* Send state */
-CONC_RECV_EVENT recv;          /* Receive state */
-CONC_WAIT_EVENT wait;          /* Wait-for-completions state */
-CONC_DELAY_EVENT delay;        /* State for spins and sleeps */
-CONC_TOUCH_EVENT touch;        /* State for memory touching */
-CONC_SYNC_EVENT sync;          /* Synchronization state */
-CONC_MCAST_EVENT mcast;        /* Multicast state */
-CONC_REDUCE_EVENT reduce;      /* Reduction state */
-CONC_BTIME_EVENT btime;        /* Timed-loop state */
-CONC_ETIME_EVENT etime;        /* Additional timed-loop state */
-CONC_REPEAT_EVENT rep;         /* Repeated-events state */
-CONC_SUPPRESS_EVENT suppress;  /* State for suppressing output */
-CONC_CODE_EVENT code;          /* State for arbitrary code */
-} s;
-} CONC_EVENT;
-
-/* Fully specify an arbitrary for() loop (used by FOR EACH). */
-typedef struct {
-NCPTL_QUEUE *list_comp;  /* NULL=ordinary list; other=list comprehension values */
-int integral;        /* 1=integral values; 0=floating-point values */
-enum {               /* Comparison of loop variable to end variable */
-CONC_LEQ,                /* Increasing progression */
-CONC_GEQ                 /* Decreasing progression */
-} comparator;
-enum {               /* How to increment the loop variable */
-CONC_ADD,                /* Arithmetically */
-CONC_MULT,               /* Geometrically increasing */
-CONC_DIV                 /* Geometrically decreasing */
-} increment;
-union {
-struct {
-ncptl_int loopvar;   /* Loop variable */
-ncptl_int prev_loopvar; /* Previous value of loop variable */
-ncptl_int startval;  /* Initial value of loop variable */
-ncptl_int endval;    /* Value not to exceed */
-ncptl_int incval;    /* Loop-variable increment */
-} i;
-struct {
-double loopvar;      /* Loop variable */
-double prev_loopvar; /* Previous value of loop variable */
-double startval;     /* Initial value of loop variable */
-double endval;       /* Value not to exceed */
-double incval;       /* Loop-variable increment */
-} d;
-} u;
-} LOOPBOUNDS;
-
-/********************
- * Global variables *
- ********************/
-
-/* Variables exported to coNCePTuaL programs */
-static ncptl_int var_bytes_received = 0;   /* Total number of bytes received */
-static ncptl_int var_msgs_received = 0;   /* Total number of messages received */
-static ncptl_int var_bit_errors = 0;   /* Total number of bit errors observed */
-static ncptl_int var_total_msgs = 0;   /* Sum of messages sent and messages received */
-static ncptl_int var_msgs_sent = 0;   /* Total number of messages sent */
-static ncptl_int var_bytes_sent = 0;   /* Total number of bytes sent */
-static ncptl_int var_num_tasks = 1;   /* Number of tasks running the program */
-static ncptl_int var_elapsed_usecs = 0;   /* Elapsed time in microseconds */
-static ncptl_int var_total_bytes = 0;   /* Sum of bytes sent and bytes received */
-
-/* Dummy variable to help mark other variables as used */
-static union {
-ncptl_int ni;
-int i;
-void *vp;
-} conc_dummy_var;
-
-/* Variables used internally by boilerplate code */
-static uint64_t starttime;   /* Time the clock was last reset (microseconds) */
-static ncptl_int pendingrecvs = 0;   /* Current # of outstanding receives */
-static ncptl_int pendingrecvbytes = 0; /* Current # of bytes in outstanding receives */
-static NCPTL_QUEUE *touchedqueue;      /* Queue of asynchronous receives to touch */
-static ncptl_int pendingsends = 0;   /* Current # of outstanding sends */
-static NCPTL_QUEUE *eventqueue;   /* List of coNCePTuaL events to perform */
-static int within_time_loop = 0;   /* 1=we're within a FOR <time> loop */
-static int suppress_output = 0;    /* 1=suppress output to stdout and the log file */
-static void *touch_region = NULL;   /* Memory region to touch */
-static ncptl_int touch_region_size = 0;   /* # of bytes in the above */
-static int virtrank;    /* This task's virtual rank in the computation */
-static int physrank;    /* This task's physical rank in the computation */
-static NCPTL_VIRT_PHYS_MAP *procmap;  /* Virtual to physical rank mapping */
-static NCPTL_LOG_FILE_STATE *logstate;   /* Opaque object representing all log-file state */
-static char *logfile_uuid;   /* Execution UUID to write to every log file */
-static char *logfiletmpl;   /* Template for the log file's name */
-//static char *logfiletmpl_default;   /* Default value of the above */
-
-/* Global variables specific to the c_mpi backend */
-static ncptl_int mpi_is_running = 0;   /* 1=MPI has been initialized */
-static NCPTL_QUEUE * recvreqQ;   /* List of MPI receive requests */
-static MPI_Request * recvrequests;   /* List version of recvreqQ */
-static NCPTL_QUEUE * recvstatQ;   /* List of MPI receive statuses */
-static MPI_Status * recvstatuses;   /* List version of recvstatQ */
-static NCPTL_QUEUE * sendreqQ;   /* List of MPI send requests */
-static MPI_Request * sendrequests;   /* List version of sendreqQ */
-static NCPTL_QUEUE * sendstatQ;   /* List of MPI send statuses */
-static MPI_Status * sendstatuses;   /* List version of sendstatQ */
-static NCPTL_SET * communicators;   /* Map from an array of processor flags to an MPI communicator */
-static MPI_Errhandler mpi_error_handler;   /* Handle to handle_MPI_error() */
-static ncptl_int mpi_tag_ub;   /* Upper bound on an MPI tag value */
-static ncptl_int conc_mcast_tallies[CONC_MCAST_MPI_NUM_FUNCS] = {0};   /* Tallies of (static) multicast implementation functions */
-
-/* Program-specific variables */
-static ncptl_int var_reps;   /* Number of repetitions of each message size (command-line argument) */
-static ncptl_int var_maxbytes;   /* Maximum number of bytes to transmit (command-line argument) */
-
-/*************************
- * Function declarations *
- *************************/
-
-/* Make MPI errors invoke ncptl_fatal(). */
-static void handle_MPI_error (MPI_Comm *comm, int *errcode, ...)
-{
-va_list args;
-char errstring[MPI_MAX_ERROR_STRING];
-int errstrlen;
-
-va_start (args, errcode);
-if (MPI_Error_string (*errcode, errstring, &errstrlen) == MPI_SUCCESS)
-ncptl_fatal ("MPI run-time error: %s", errstring);
-else
-ncptl_fatal ("MPI aborted with unrecognized error code %d", *errcode);
-conc_dummy_var.vp = (void *) comm;   /* Prevent the compiler from complaining that comm is unused. */
-va_end (args);
-}
-
-/* Perform the equivalent of CODES_MPI_Comm_rank() for an arbitrary process. */
-static int rank_in_MPI_communicator (MPI_Comm subcomm, int global_rank)
-{
-  MPI_Group world_group;   /* Group associated with MPI_COMM_WORLD */
-  MPI_Group subgroup;      /* Group associate with subcomm */
-  int subrank;             /* global_rank's rank within subcomm */
-
-  MPI_Comm_group (MPI_COMM_WORLD, &world_group);
-  MPI_Comm_group (subcomm, &subgroup);
-  MPI_Group_translate_ranks (world_group, 1, &global_rank, subgroup, &subrank);
-  return subrank;
-}
-
-/* Map an arbitrary tag to within MPI's valid range of [0, mpi_tag_ub]. */
-static ncptl_int map_tag_into_MPI_range (ncptl_int tag)
-{
-if (tag == NCPTL_INT_MIN)
- /* Avoid taking the absolute value of NCPTL_INT_MIN. */
-tag = 555666773LL;   /* Arbitrary value */
-tag = ncptl_func_abs (tag);   /* Only nonnegatives values are allowed. */
-if (mpi_tag_ub < NCPTL_INT_MAX)
-tag %= mpi_tag_ub + 1;
-return tag;
-}
-
-/* Given an array of task in/out booleans return an MPI
- * communicator that represents the "in" tasks. */
-static MPI_Comm define_MPI_communicator (char *procflags)
-{
-MPI_Comm *existing_comm;    /* Previously defined MPI communicator */
-MPI_Comm new_comm;          /* Newly defined MPI communicator */
-
-existing_comm = (MPI_Comm *) ncptl_set_find (communicators, (void *)procflags);
-if (existing_comm)
-return *existing_comm;
-(void) MPI_Comm_split (MPI_COMM_WORLD, (int)procflags[physrank], physrank, &new_comm);
-//(void) MPI_Errhandler_set (new_comm, mpi_error_handler);
-ncptl_set_insert (communicators, (void *)procflags, (void *)&new_comm);
-return define_MPI_communicator (procflags);
-}
-
-/* Return 1 if a sequence loop will take at least one trip. */
-static int conc_seq_nonempty (LOOPBOUNDS *seq)
-{
-ncptl_int startval;   /* Integer version of seq's startval element */
-ncptl_int endval;   /* Integer version of seq's endval element */
-
-if (seq->integral) {
-startval = seq->u.i.startval;
-endval = seq->u.i.endval;
-}
-else {
-startval = CONC_DBL2INT (seq->u.d.startval);
-endval = CONC_DBL2INT (seq->u.d.endval);
-}
-switch (seq->comparator) {
-case CONC_LEQ:
-return startval <= endval;
-
-case CONC_GEQ:
-return startval >= endval;
-
-default:
-ncptl_fatal ("Internal error -- unknown comparator");
-}
-return -1;     /* Appease idiotic compilers. */
-}
-
-/* Initialize a sequence loop. */
-static void conc_seq_init (LOOPBOUNDS *seq)
-{
-if (seq->integral) {
-seq->u.i.loopvar = seq->u.i.startval;
-seq->u.i.prev_loopvar = seq->u.i.loopvar - 1;
-}
-else {
-seq->u.d.loopvar = seq->u.d.startval;
-seq->u.d.prev_loopvar = seq->u.d.loopvar - 1.0;
-}
-}
-
-/* Return 1 if a sequence loop should continue, 0 when finished. */
-static int conc_seq_continue (LOOPBOUNDS *seq)
-{
-LOOPBOUNDS seq_int;   /* Integer equivalent of *seq */
-if (seq->integral)
-seq_int = *seq;
-else {
-seq_int.u.i.loopvar = CONC_DBL2INT (seq->u.d.loopvar);
-seq_int.u.i.prev_loopvar = CONC_DBL2INT (seq->u.d.prev_loopvar);
-seq_int.u.i.endval = CONC_DBL2INT (seq->u.d.endval);
-}
-
-if (seq_int.u.i.loopvar == seq_int.u.i.prev_loopvar)
-return 0;
-switch (seq->comparator) {
-case CONC_LEQ:
-return seq_int.u.i.loopvar <= seq_int.u.i.endval;
-
-case CONC_GEQ:
-return seq_int.u.i.loopvar >= seq_int.u.i.endval;
-
-default:
-ncptl_fatal ("Internal error -- unknown comparator");
-}
-return -1;     /* Appease idiotic compilers. */
-}
-
-/* Proceed to the next iteration of a sequence loop. */
-static void conc_seq_next (LOOPBOUNDS *seq)
-{
-if (seq->integral) {
-seq->u.i.prev_loopvar = seq->u.i.loopvar;
-switch (seq->increment) {
-case CONC_ADD:
-seq->u.i.loopvar += seq->u.i.incval;
-break;
-
-case CONC_MULT:
-seq->u.i.loopvar *= seq->u.i.incval;
-break;
-
-case CONC_DIV:
-seq->u.i.loopvar /= seq->u.i.incval;
-break;
-
-default:
-ncptl_fatal ("Internal error -- unknown incrementer");
-}
-}
-else {
-seq->u.d.prev_loopvar = seq->u.d.loopvar;
-switch (seq->increment) {
-case CONC_ADD:
-seq->u.d.loopvar += seq->u.d.incval;
-break;
-
-case CONC_MULT:
-seq->u.d.loopvar *= seq->u.d.incval;
-break;
-
-case CONC_DIV:
-seq->u.d.loopvar /= seq->u.d.incval;
-break;
-
-default:
-ncptl_fatal ("Internal error -- unknown incrementer");
-}
-}
-}
-
-/* Inhibit the compiler from complaining that
- * certain variables are defined but not used.
- * This function should never be called. */
-static void conc_mark_variables_used (void)
-{
-conc_dummy_var.ni = var_bytes_received;
-conc_dummy_var.ni = var_msgs_received;
-conc_dummy_var.ni = var_bit_errors;
-conc_dummy_var.ni = var_total_msgs;
-conc_dummy_var.ni = var_msgs_sent;
-conc_dummy_var.ni = var_bytes_sent;
-conc_dummy_var.ni = var_num_tasks;
-conc_dummy_var.ni = var_elapsed_usecs;
-conc_dummy_var.ni = var_total_bytes;
-conc_dummy_var.ni = pendingrecvbytes;
-conc_dummy_var.ni = touch_region_size;
-conc_dummy_var.vp = touch_region;
-conc_dummy_var.i  = within_time_loop;
-conc_dummy_var.i  = suppress_output;
-rank_in_MPI_communicator (MPI_COMM_WORLD, 0);
-}
-
-/* Allocate a new event of a given type and return a pointer to it. */
-static CONC_EVENT *conc_allocate_event (CONC_EVENT_TYPE type)
-{
-CONC_EVENT *newevent = (CONC_EVENT *) ncptl_queue_allocate (eventqueue);
-
-newevent->type = type;
-return newevent;
-}
-
-/* Declare an exit handler that gets called automatically when the
- * program terminates, whether successfully or not. */
-static void conc_exit_handler (void)
-{
-if (mpi_is_running)
-MPI_Abort (MPI_COMM_WORLD, 1);
-}
-
-
-/* Initialize coNCePTuaL, the messaging layer, and this program itself. */
-static void conc_initialize (int argc, char *argv[])
-{
- /* Variables needed by all C-based backends */
-CONC_EVENT * eventlist;   /* List of events to execute */
-ncptl_int numevents;   /* Number of entries in eventlist[] */
-int help_only = 0;   /* 1=User specified --help; save time by skipping ncptl_init() */
-char * argv0 = strrchr(argv[0], '/') ? strrchr(argv[0], '/')+1 : argv[0];   /* Base name of the executable program */
-int i;   /* Generic loop variable */
-
- /* Declare all of our command-line arguments. */
-NCPTL_CMDLINE arguments[] = {
-{ NCPTL_TYPE_STRING, NULL, "logfile", 'L', "Log-file template", {0}},
-{ NCPTL_TYPE_INT, NULL, "reps", 'r', "Number of repetitions of each message size", {0}},
-{ NCPTL_TYPE_INT, NULL, "maxbytes", 'm', "Maximum number of bytes to transmit", {0}}
-};
-
- /* Incorporate the complete coNCePTuaL source code as an array
-//  * for use by ncptl_log_write_prologue(). */
-char *sourcecode[] = {
-"# An all-pairs ping-pong latency test written in coNCePTuaL",
-"# By Scott Pakin <pakin@lanl.gov>",
-"",
-"Require language version \"1.5\".",
-"",
-"# Parse the command line.",
-"reps is \"Number of repetitions of each message size\" and comes from \"--reps\" or \"-r\" with default 1000.",
-"maxbytes is \"Maximum number of bytes to transmit\" and comes from \"--maxbytes\" or \"-m\" with default 1M.",
-"",
-"# Ensure that we have a peer with whom to communicate.",
-"Assert that \"the latency test requires at least two tasks\" with num_tasks>=2.",
-"",
-"# Perform the benchmark.",
-"For each msgsize in {0}, {1, 2, 4, ..., maxbytes} {",
-"  for reps repetitions {",
-"    tasks ev such that ev is even reset their counters then",
-"    tasks ev such that ev is even send a msgsize byte message to task ev+1 then",
-"    tasks od such that od is odd send a msgsize byte message to task od-1 then",
-"    tasks ev such that ev is even log the msgsize as \"Bytes\" and",
-"                                      the median of elapsed_usecs/2 as \"1/2 RTT (usecs)\"",
-"  } then",
-"  tasks ev such that ev is even compute aggregates",
-"}",
-NULL
-};
-
- /* Variables specific to the c_mpi backend */
-int num_tasks;   /* int version of var_num_tasks needed by CODES_MPI_Comm_size() */
-char * procflags;   /* Array of 1s representing an all-task MPI communicator */
-MPI_Comm comm_world = MPI_COMM_WORLD;   /* Copy of MPI_COMM_WORLD that we can take the address of */
-void * attr_val;   /* Pointed to the value of MPI_TAG_UB */
-int attr_flag = 0;   /* true=MPI_TAG_UB was extracted; false=not extracted */
-char log_key_str[128];   /* String representing the range of valid MPI tags */
-
- /* As a special case, if the command line contains --help, then skip
-  * the coNCePTuaL initialization step. */
-for (i=1; i<argc; i++)
-if (!strcmp(argv[i], "--"))
-break;
-else
-if (!strcmp(argv[i], "--help") || !strcmp(argv[i], "-?")) {
-argv[1] = "-?";   /* Guaranteed to work, even with getopt() */
-help_only = 1;
-break;
-}
-
- /* Perform various initializations specific to the c_mpi backend. */
- /* Initialize MPI. */
-//(void) MPI_Init(&argc, &argv);
-mpi_is_running = 1;
-
- /* Initialize the coNCePTuaL run-time library. */
-if (!help_only)
-//ncptl_init (NCPTL_RUN_TIME_VERSION, argv[0]);
-//(void) atexit (conc_exit_handler);
-
- /* Initialize the communication routines needed by the c_mpi backend. */
-//(void) MPI_Errhandler_create ((MPI_Handler_function *)handle_MPI_error, &mpi_error_handler);
-//(void) MPI_Errhandler_set (MPI_COMM_WORLD, mpi_error_handler);
-(void) CODES_MPI_Comm_rank(MPI_COMM_WORLD, &physrank);
-(void) CODES_MPI_Comm_size(MPI_COMM_WORLD, &num_tasks);
-var_num_tasks = (ncptl_int) num_tasks;
-//(void) MPI_Comm_get_attr(MPI_COMM_WORLD, MPI_TAG_UB, &attr_val, &attr_flag);
-mpi_tag_ub = (ncptl_int) (attr_flag ? *(int *)attr_val : 32767);
-
- /* Generate and broadcast a UUID. */
-//logfile_uuid = ncptl_log_generate_uuid();
-//(void) CODES_MPI_Bcast ((void *)logfile_uuid, 37, MPI_CHAR, 0, MPI_COMM_WORLD);
-
- /* Plug variables and default values into the NCPTL_CMDLINE structure. */
-arguments[0].variable = (CMDLINE_VALUE *) &logfiletmpl;
-arguments[1].variable = (CMDLINE_VALUE *) &var_reps;
-arguments[1].defaultvalue.intval = 1000LL;
-arguments[2].variable = (CMDLINE_VALUE *) &var_maxbytes;
-arguments[2].defaultvalue.intval = 1048576LL;
-//logfiletmpl_default = (char *) ncptl_malloc (strlen(argv0) + 15, 0);
-//sprintf (logfiletmpl_default, "%s-%%p.log", argv0);
-//arguments[0].defaultvalue.stringval = logfiletmpl_default;
-
- /* Parse the command line. */
-mpi_is_running = 0;   /* Don't invoke MPI_Abort() after --help. */
-ncptl_parse_command_line (argc, argv, arguments, sizeof(arguments)/sizeof(NCPTL_CMDLINE));
-if (help_only)
-ncptl_fatal ("Internal error in the c_generic backend: failed to exit after giving help");
-mpi_is_running = 1;
-
- /* Establish a mapping from (virtual) task IDs to (physical) ranks. */
-procmap = ncptl_allocate_task_map (var_num_tasks);
-virtrank = ncptl_physical_to_virtual (procmap, physrank);
-
- /* Perform initializations specific to the c_mpi backend. */
-//ncptl_log_add_comment ("MPI send routines", "MPI_Send() and CODES_MPI_Isend()");
-//ncptl_log_add_comment ("MPI reduction operation", REDUCE_OPERATION_NAME);
-//sprintf (log_key_str, "[0, %" NICS "]", mpi_tag_ub);
-//ncptl_log_add_comment ("MPI tag range", log_key_str);
-//
-// /* Open the log file and write some standard prologue information to it. */
-//logstate = ncptl_log_open (logfiletmpl, physrank);
-////ncptl_log_write_prologue (logstate, argv[0], logfile_uuid, "c_mpi", "C + MPI",
-//var_num_tasks,
-//arguments, sizeof(arguments)/sizeof(NCPTL_CMDLINE),
-//sourcecode);
-//ncptl_free (logfile_uuid);
-
- /* Allocate a variety of dynamically growing queues. */
-eventqueue = ncptl_queue_init (sizeof (CONC_EVENT));
-touchedqueue = ncptl_queue_init (sizeof (ncptl_int));
-
- /* Perform initializations specific to the c_mpi backend. */
-sendreqQ = ncptl_queue_init (sizeof (MPI_Request));
-sendstatQ = ncptl_queue_init (sizeof (MPI_Status));
-recvreqQ = ncptl_queue_init (sizeof (MPI_Request));
-recvstatQ = ncptl_queue_init (sizeof (MPI_Status));
-communicators = ncptl_set_init (ESTIMATED_COMMUNICATORS, var_num_tasks*sizeof(char), sizeof(MPI_Comm));procflags = (char *) ncptl_malloc (var_num_tasks*sizeof(char), 0);
-for (i=0; i<var_num_tasks; i++)
-procflags[i] = 1;
-ncptl_set_insert (communicators, (void *)procflags, (void *)&comm_world);
-ncptl_free (procflags);
-
- /****************************************************
-  * Generated, program-specific initialization code. *
-  ****************************************************/
-
- /* ASSERT THAT the latency test requires at least two tasks. */
-if (!((var_num_tasks)>=(2LL)))
-ncptl_fatal ("Assertion failure: the latency test requires at least two tasks");
- /* FOR EACH var_msgsize IN [(['0LL'], None), (['1LL', '2LL', '4LL'], 'var_maxbytes')]... */
-{
-LOOPBOUNDS loopbounds[2];   /* List of range descriptions */
-ncptl_int rangenum;   /* Current offset into loopbounds[] */
-ncptl_int initial_vals[3];   /* Cache of the initial, enumerated values */
-ncptl_int final_val;   /* Cache of the final value */
-
- /* Write range 0's loop bounds, "next" function, and termination function to loopbounds[0]. */
-loopbounds[0].list_comp = NULL;
-initial_vals[0] = 0LL;
-final_val = 0LL;
-loopbounds[0].integral = 1;
-loopbounds[0].u.i.startval = initial_vals[0];
-loopbounds[0].u.i.endval = final_val;
-loopbounds[0].comparator = CONC_LEQ;
-loopbounds[0].increment = CONC_ADD;
-loopbounds[0].u.i.incval = 1LL;
-
- /* Write range 1's loop bounds, "next" function, and termination function to loopbounds[1]. */
-loopbounds[1].list_comp = NULL;
-initial_vals[0] = 1LL;
-initial_vals[1] = 2LL;
-initial_vals[2] = 4LL;
-final_val = var_maxbytes;
-loopbounds[1].integral = 1;
-loopbounds[1].u.i.startval = initial_vals[0];
-loopbounds[1].u.i.endval = final_val;
-if (initial_vals[1]-initial_vals[0]==initial_vals[2]-initial_vals[1]) {
- /* Arithmetic progression */
-loopbounds[1].comparator = initial_vals[0]<=initial_vals[1] ? CONC_LEQ : CONC_GEQ;
-loopbounds[1].increment = CONC_ADD;
-loopbounds[1].u.i.incval = initial_vals[1]-initial_vals[0];
-if (!loopbounds[1].u.i.incval)
-loopbounds[1].u.i.incval = 1LL;   /* Handle {x,x,x,...,x} case (constant value) */
-}
-else
-if (initial_vals[0] && initial_vals[1] && initial_vals[0]<initial_vals[1] && initial_vals[0]*(initial_vals[1]/initial_vals[0])==initial_vals[1] && initial_vals[1]*(initial_vals[2]/initial_vals[1])==initial_vals[2] && initial_vals[1]/initial_vals[0]==initial_vals[2]/initial_vals[1]) {
- /* Geometric progression (increasing, integral multiplier) */
-loopbounds[1].comparator = CONC_LEQ;
-loopbounds[1].increment = CONC_MULT;
-loopbounds[1].u.i.incval = initial_vals[1]/initial_vals[0];
-}
-else
-if (initial_vals[1] && initial_vals[2] && initial_vals[0]>initial_vals[1] && initial_vals[1]*(initial_vals[0]/initial_vals[1])==initial_vals[0] && initial_vals[2]*(initial_vals[1]/initial_vals[2])==initial_vals[1] && initial_vals[0]/initial_vals[1]==initial_vals[1]/initial_vals[2]) {
- /* Geometric progression (decreasing, integral multiplier) */
-loopbounds[1].comparator = CONC_GEQ;
-loopbounds[1].increment = CONC_DIV;
-loopbounds[1].u.i.incval = initial_vals[0]/initial_vals[1];
-}
-else
-if (initial_vals[0] && initial_vals[1]) {
-double initial_vals_d[3];   /* Cache of the initial, enumerated values, but in floating-point context */
-double avg_factor;   /* Average multiplier for terms in the sequence */
-initial_vals_d[0] = 1.0;
-initial_vals_d[1] = 2.0;
-initial_vals_d[2] = 4.0;
-avg_factor = (initial_vals_d[1]/initial_vals_d[0] + initial_vals_d[2]/initial_vals_d[1]) / 2.0;
-if (CONC_DBL2INT(initial_vals_d[0]*avg_factor)==initial_vals[1] && CONC_DBL2INT(initial_vals_d[1]*avg_factor)==initial_vals[2]) {
- /* Geometric progression (decreasing or non-integral multiplier) */
-loopbounds[1].comparator = initial_vals[0]<initial_vals[1] ? CONC_LEQ : CONC_GEQ;
-loopbounds[1].integral = 0;
-loopbounds[1].increment = CONC_MULT;
-loopbounds[1].u.d.startval = initial_vals_d[0];
-loopbounds[1].u.d.endval = (double)var_maxbytes;
-loopbounds[1].u.d.incval = avg_factor;
-}
-else
-ncptl_fatal ("Unable to find an arithmetic or geometric pattern to {%" NICS ", %" NICS ", %" NICS ", ..., %" NICS "}", initial_vals[0], initial_vals[1], initial_vals[2], final_val);
-}
-else
-ncptl_fatal ("Unable to find an arithmetic or geometric pattern to {%" NICS ", %" NICS ", %" NICS ", ..., %" NICS "}", initial_vals[0], initial_vals[1], initial_vals[2], final_val);
-
- /* Now that we've defined all of our ranges we iterate over each range
-  * and each element within each range. */
-for (rangenum=0LL; rangenum<2; rangenum++) {
-LOOPBOUNDS * thisrange = &loopbounds[rangenum];   /* Current range */
-if (conc_seq_nonempty (thisrange))
-for (conc_seq_init (thisrange);
-conc_seq_continue (thisrange);
-conc_seq_next (thisrange)) {
-ncptl_int var_msgsize;
-if (thisrange->list_comp == NULL)
- /* The loopvar field contains the value for var_msgsize. */
-var_msgsize = thisrange->integral ? thisrange->u.i.loopvar : CONC_DBL2INT(thisrange->u.d.loopvar);
-else
- /* The queue of list-comprehension values provides the value for var_msgsize. */
-var_msgsize = *(ncptl_int *)ncptl_queue_pop(thisrange->list_comp);
-{
- /* FOR var_reps REPETITIONS... */
-{
-ncptl_int numreps = var_reps;   /* Total # of repetitions to perform */
-int unroll_loop = numreps <= CONC_MAX_UNROLL;   /* 1=unroll loop; 0=use a REPEAT event */
-ncptl_int repnum;   /* Current repetition number */
-CONC_EVENT * repevent = NULL;   /* Event designating repetition */
-ncptl_int repeventnum = -1LL;   /* Event number corresponding to repevent */
- /* Conditionally unroll the loop. */
-for (repnum=0; repnum<(unroll_loop?numreps:1); repnum++) {
- /* Allocate a repeat event if we have more than one repetition. */
-if (!unroll_loop && numreps > 1LL) {
-repeventnum = ncptl_queue_length (eventqueue);
-repevent =  conc_allocate_event (EV_REPEAT);
-repevent->s.rep.numreps = numreps;
-}
-
- /* Output a loop body if we have at least one repetition. */
-if (unroll_loop || numreps > 0LL) {
- /* TASKS var_ev SUCH THAT ((var_ev)&1)==0 RESET THEIR COUNTERS */
-{
-ncptl_int var_ev = virtrank;
-if (((var_ev)&1)==0) {   /* TASKS var_ev SUCH THAT ((var_ev)&1)==0 */
- /* The current coNCePTuaL statement applies to our task. */
-(void) conc_allocate_event (EV_RESET);
-}
-}
- /* THEN... */
- /* TASKS var_ev SUCH THAT ((var_ev)&1)==0 RECEIVES FROM TASK (var_ev)+(1LL) */
-{
-ncptl_int var_ev;
- /* Loop over all tasks to see which will send to us. */
-for (var_ev=0; var_ev<var_num_tasks; var_ev++)
-if (((var_ev)&1)==0) {
- /* var_ev now represents one of the tasks that will send to us. */
-ncptl_int ivar_a_task = var_ev;
-if (virtrank == ((var_ev)+(1LL))) {
- /* In this scope, we must be a message recipient. */
- /* Prepare to receive 1LL messages. */
-ncptl_int numreps = 1LL;   /* Number of messages */
-if (numreps > 1LL) {
-CONC_EVENT * repeatev;   /* Event specifying the number of repetitions to perform */
-repeatev = conc_allocate_event (EV_REPEAT);
-repeatev->s.rep.end_event = ncptl_queue_length(eventqueue);
-repeatev->s.rep.numreps = numreps;
-}
-
- /* Ensure we have at least one message to receive. */
-if (numreps > 0LL) {
-CONC_EVENT *thisev = conc_allocate_event (EV_RECV);
-
- /* Fill in all of the fields of a receive-event structure. */
-thisev->s.recv.source = ncptl_virtual_to_physical (procmap, ivar_a_task);
-thisev->s.recv.size = var_msgsize;
-thisev->s.recv.alignment = 0LL;
-thisev->s.recv.misaligned = 0;
-thisev->s.recv.touching = 0;
-thisev->s.recv.verification = 0;
-thisev->s.recv.tag = 0LL;
-thisev->s.recv.pendingsends = pendingsends;
-thisev->s.recv.pendingrecvs = pendingrecvs;
-thisev->s.recv.bufferofs = 0LL;
-thisev->s.recv.buffernum = thisev->s.recv.verification ? pendingsends+pendingrecvs : 0;
-(void) ncptl_malloc_message (thisev->s.recv.size+thisev->s.recv.bufferofs, thisev->s.recv.alignment, thisev->s.recv.buffernum, thisev->s.recv.misaligned);
-thisev->s.recv.buffer = NULL;
-thisev->s.recv.tag = map_tag_into_MPI_range (thisev->s.recv.tag);
-}
-}
-}
-}
- /* TASKS var_ev SUCH THAT ((var_ev)&1)==0 SENDS TO TASK (var_ev)+(1LL) */
-{
-ncptl_int var_ev = virtrank;
-if (((var_ev)&1)==0) {   /* TASKS var_ev SUCH THAT ((var_ev)&1)==0 */
- /* The current coNCePTuaL statement applies to our task. */
-ncptl_int virtdest = (var_ev)+(1LL);
-if (virtdest>=0 && virtdest<var_num_tasks) {
- /* In this scope, (var_ev)+(1LL) represents a single receiver. */
- /* Prepare to send 1LL messages. */
-ncptl_int numreps = 1LL;   /* Number of messages */
-if (numreps > 1LL) {
-CONC_EVENT * repeatev;   /* Event specifying the number of repetitions to perform */
-repeatev = conc_allocate_event (EV_REPEAT);
-repeatev->s.rep.end_event = ncptl_queue_length(eventqueue);
-repeatev->s.rep.numreps = numreps;
-}
-
- /* Ensure we have at least one message to send. */
-if (numreps > 0LL) {
-CONC_EVENT *thisev = conc_allocate_event (EV_SEND);
-if (virtrank == ((var_ev)+(1LL)))
-ncptl_fatal ("Send-to-self deadlock encountered on task %d in line 17 of the source code", virtrank);
-
- /* Fill in all of the fields of a send-event structure. */
-thisev->s.send.dest = ncptl_virtual_to_physical (procmap, (var_ev)+(1LL));
-thisev->s.send.size = var_msgsize;
-thisev->s.send.alignment = 0LL;
-thisev->s.send.misaligned = 0;
-thisev->s.send.touching = 0;
-thisev->s.send.verification = 0;
-thisev->s.send.tag = 0LL;
-thisev->s.send.pendingsends = pendingsends;
-thisev->s.send.pendingrecvs = pendingrecvs;
-thisev->s.send.bufferofs = 0LL;
-thisev->s.send.buffernum = thisev->s.send.verification ? pendingsends+pendingrecvs : 0;
-(void) ncptl_malloc_message (thisev->s.send.size+thisev->s.send.bufferofs, thisev->s.send.alignment, thisev->s.send.buffernum, thisev->s.send.misaligned);
-thisev->s.send.buffer = NULL;
-thisev->s.send.tag = map_tag_into_MPI_range (thisev->s.send.tag);
-}
-}
-}
-}
- /* THEN... */
- /* TASKS var_od SUCH THAT ((var_od)&1)==1 RECEIVES FROM TASK (var_od)-(1LL) */
-{
-ncptl_int var_od;
- /* Loop over all tasks to see which will send to us. */
-for (var_od=0; var_od<var_num_tasks; var_od++)
-if (((var_od)&1)==1) {
- /* var_od now represents one of the tasks that will send to us. */
-ncptl_int ivar_b_task = var_od;
-if (virtrank == ((var_od)-(1LL))) {
- /* In this scope, we must be a message recipient. */
- /* Prepare to receive 1LL messages. */
-ncptl_int numreps = 1LL;   /* Number of messages */
-if (numreps > 1LL) {
-CONC_EVENT * repeatev;   /* Event specifying the number of repetitions to perform */
-repeatev = conc_allocate_event (EV_REPEAT);
-repeatev->s.rep.end_event = ncptl_queue_length(eventqueue);
-repeatev->s.rep.numreps = numreps;
-}
-
- /* Ensure we have at least one message to receive. */
-if (numreps > 0LL) {
-CONC_EVENT *thisev = conc_allocate_event (EV_RECV);
-
- /* Fill in all of the fields of a receive-event structure. */
-thisev->s.recv.source = ncptl_virtual_to_physical (procmap, ivar_b_task);
-thisev->s.recv.size = var_msgsize;
-thisev->s.recv.alignment = 0LL;
-thisev->s.recv.misaligned = 0;
-thisev->s.recv.touching = 0;
-thisev->s.recv.verification = 0;
-thisev->s.recv.tag = 0LL;
-thisev->s.recv.pendingsends = pendingsends;
-thisev->s.recv.pendingrecvs = pendingrecvs;
-thisev->s.recv.bufferofs = 0LL;
-thisev->s.recv.buffernum = thisev->s.recv.verification ? pendingsends+pendingrecvs : 0;
-(void) ncptl_malloc_message (thisev->s.recv.size+thisev->s.recv.bufferofs, thisev->s.recv.alignment, thisev->s.recv.buffernum, thisev->s.recv.misaligned);
-thisev->s.recv.buffer = NULL;
-thisev->s.recv.tag = map_tag_into_MPI_range (thisev->s.recv.tag);
-}
-}
-}
-}
- /* TASKS var_od SUCH THAT ((var_od)&1)==1 SENDS TO TASK (var_od)-(1LL) */
-{
-ncptl_int var_od = virtrank;
-if (((var_od)&1)==1) {   /* TASKS var_od SUCH THAT ((var_od)&1)==1 */
- /* The current coNCePTuaL statement applies to our task. */
-ncptl_int virtdest = (var_od)-(1LL);
-if (virtdest>=0 && virtdest<var_num_tasks) {
- /* In this scope, (var_od)-(1LL) represents a single receiver. */
- /* Prepare to send 1LL messages. */
-ncptl_int numreps = 1LL;   /* Number of messages */
-if (numreps > 1LL) {
-CONC_EVENT * repeatev;   /* Event specifying the number of repetitions to perform */
-repeatev = conc_allocate_event (EV_REPEAT);
-repeatev->s.rep.end_event = ncptl_queue_length(eventqueue);
-repeatev->s.rep.numreps = numreps;
-}
-
- /* Ensure we have at least one message to send. */
-if (numreps > 0LL) {
-CONC_EVENT *thisev = conc_allocate_event (EV_SEND);
-if (virtrank == ((var_od)-(1LL)))
-ncptl_fatal ("Send-to-self deadlock encountered on task %d in line 18 of the source code", virtrank);
-
- /* Fill in all of the fields of a send-event structure. */
-thisev->s.send.dest = ncptl_virtual_to_physical (procmap, (var_od)-(1LL));
-thisev->s.send.size = var_msgsize;
-thisev->s.send.alignment = 0LL;
-thisev->s.send.misaligned = 0;
-thisev->s.send.touching = 0;
-thisev->s.send.verification = 0;
-thisev->s.send.tag = 0LL;
-thisev->s.send.pendingsends = pendingsends;
-thisev->s.send.pendingrecvs = pendingrecvs;
-thisev->s.send.bufferofs = 0LL;
-thisev->s.send.buffernum = thisev->s.send.verification ? pendingsends+pendingrecvs : 0;
-(void) ncptl_malloc_message (thisev->s.send.size+thisev->s.send.bufferofs, thisev->s.send.alignment, thisev->s.send.buffernum, thisev->s.send.misaligned);
-thisev->s.send.buffer = NULL;
-thisev->s.send.tag = map_tag_into_MPI_range (thisev->s.send.tag);
-}
-}
-}
-}
- /* THEN... */
- /* TASKS var_ev SUCH THAT ((var_ev)&1)==0 LOGS "Bytes" AND "1/2 RTT (usecs)" */
-{
-ncptl_int var_ev = virtrank;
-if (((var_ev)&1)==0) {   /* TASKS var_ev SUCH THAT ((var_ev)&1)==0 */
- /* The current coNCePTuaL statement applies to our task. */
-CONC_EVENT *thisev = conc_allocate_event (EV_CODE);
-thisev->s.code.number = 0;
-thisev->s.code.procmap = NULL;
-thisev->s.code.var_msgsize = var_msgsize;
-}
-}
-}
-}
-
- /* Assign the number of events to repeat, now that we know that number. */
-if (!unroll_loop && numreps > 1LL) {
-repevent = repeventnum + (CONC_EVENT *) ncptl_queue_contents (eventqueue, 0);
-repevent->s.rep.end_event = ncptl_queue_length (eventqueue) - 1;
-}
-}
- /* THEN... */
- /* TASKS var_ev SUCH THAT ((var_ev)&1)==0 COMPUTES AGGREGATES */
-{
-ncptl_int var_ev = virtrank;
-if (((var_ev)&1)==0) {   /* TASKS var_ev SUCH THAT ((var_ev)&1)==0 */
- /* The current coNCePTuaL statement applies to our task. */
-(void) conc_allocate_event (EV_FLUSH);
-}
-}
-}
-}
-}
-}
- /* Begin a new top-level statement. */
-(void) conc_allocate_event (EV_NEWSTMT);
-
- /*************************
-  * More boilerplate code *
-  *************************/
-
- /* Abort if the program will terminate with pending messages. */
-if (pendingsends && pendingrecvs)
-ncptl_fatal("Neglected to await the completion of %" NICS " asynchronous %s and %" NICS " asynchronous %s",
-pendingsends, pendingsends==1LL ? "send" : "sends",
-pendingrecvs, pendingrecvs==1LL ? "receive" : "receives");
-else
-if (pendingsends)
-ncptl_fatal("Neglected to await the completion of %" NICS " asynchronous %s",
-pendingsends, pendingsends==1LL ? "send" : "sends");
-else
-if (pendingrecvs)
-ncptl_fatal("Neglected to await the completion of %" NICS " asynchronous %s",
-pendingrecvs, pendingrecvs==1LL ? "receive" : "receives");
-
- /* Allocate memory for non-unique messages and asynchronous
-  * message handles now that we know how much memory we need
-  * to allocate. */
-eventlist = (CONC_EVENT *) ncptl_queue_contents (eventqueue, 0);
-numevents = ncptl_queue_length (eventqueue);
-sendrequests = (MPI_Request *) ncptl_queue_contents (sendreqQ, 0);
-recvrequests = (MPI_Request *) ncptl_queue_contents (recvreqQ, 0);
-for (i=0; i<numevents; i++) {
-CONC_EVENT *thisev = &eventlist[i];   /* Cache of the current event */
-switch (thisev->type) {
-case EV_SEND:
-if (!thisev->s.send.buffer)
-thisev->s.send.buffer = ncptl_malloc_message (thisev->s.send.bufferofs + thisev->s.send.size,
-thisev->s.send.alignment,
-thisev->s.send.buffernum,
-thisev->s.send.misaligned);
-if (thisev->s.send.verification)
-ncptl_fill_buffer (CONC_GETBUFPTR(send), thisev->s.send.bufferofs + thisev->s.send.size, -1);
-break;
-
-case EV_RECV:
-if (!thisev->s.recv.buffer)
-thisev->s.recv.buffer = ncptl_malloc_message (thisev->s.recv.bufferofs + thisev->s.recv.size,
-thisev->s.recv.alignment,
-thisev->s.recv.buffernum,
-thisev->s.recv.misaligned);
-if (thisev->s.recv.verification)
-ncptl_fill_buffer (CONC_GETBUFPTR(recv), thisev->s.recv.bufferofs + thisev->s.recv.size, -1);
-break;
-
-default:
-break;
-}
-}
-}
-
-/* Process a subset of the events in a given event list. */
-static void conc_process_events (CONC_EVENT *eventlist,
-ncptl_int firstev, ncptl_int lastev, ncptl_int numreps)
-{
-CONC_EVENT * thisev;   /* Cache of the current event */
-CONC_EVENT * thisev_first = &eventlist[firstev];   /* Cache of the first event */
-ncptl_int i;   /* Iterate over events. */
-ncptl_int j;   /* Iterate over repetitions. */
-
- /* Declarations specific to the c_mpi backend */
-MPI_Status status;   /* Not needed but required by CODES_MPI_Recv() */
-
- /* Process from event firstev to event lastev (both inclusive). */
-for (j=numreps; j>0; j--)
-for (i=firstev, thisev=thisev_first; i<=lastev; i++, thisev++) {
- /* Declare variables needed by all C-based backends. */
-
- /* Process a single event. */
-switch (thisev->type) {
-case EV_SEND:
- /* Synchronous send */
-(void) CODES_MPI_Send (CONC_GETBUFPTR(send),
-(int)thisev->s.send.size, MPI_BYTE,
-(int)thisev->s.send.dest, (int)thisev->s.send.tag, MPI_COMM_WORLD);
-break;
-
-case EV_RECV:
- /* Synchronous receive */
-(void) CODES_MPI_Recv (CONC_GETBUFPTR(recv),
-(int)thisev->s.recv.size, MPI_BYTE,
-(int)thisev->s.recv.source, (int)thisev->s.recv.tag,
-MPI_COMM_WORLD, &status);
-break;
-
-case EV_RESET:
- /* Reset all of the counters exported to coNCePTuaL programs. */
-starttime = ncptl_time();
-break;
-
-case EV_FLUSH:
- /* Force all aggregate functions to produce a result. */
-if (!suppress_output) {
-uint64_t stop_elapsed_usecs = ncptl_time();
-var_elapsed_usecs = stop_elapsed_usecs - starttime;
-//ncptl_log_compute_aggregates (logstate);
-starttime += ncptl_time() - stop_elapsed_usecs;
-}
-break;
-
-case EV_REPEAT:
- /* Repeatedly perform the next batch of events. */
-conc_process_events (eventlist, i+1, thisev->s.rep.end_event, thisev->s.rep.numreps);
-i = thisev->s.rep.end_event;
-thisev = &eventlist[i];
-break;
-
-case EV_NEWSTMT:
- /* Begin a new table in the log file. */
-if (!suppress_output) {
-uint64_t stop_elapsed_usecs = ncptl_time();
-var_elapsed_usecs = stop_elapsed_usecs - starttime;
-//ncptl_log_commit_data (logstate);
-starttime += ncptl_time() - stop_elapsed_usecs;
-}
-break;
-
-case EV_CODE:
- /* Execute an arbitrary piece of code. */
-switch (thisev->s.code.number) {
-case 0:
- /* TASKS var_ev SUCH THAT ((var_ev)&1)==0 LOGS "Bytes" AND "1/2 RTT (usecs)" */
-if (!suppress_output) {
-uint64_t stop_elapsed_usecs = ncptl_time();
-var_elapsed_usecs = stop_elapsed_usecs - starttime;
-//ncptl_log_write (logstate, 0, "Bytes", NCPTL_FUNC_ONLY, 0.0, (double)thisev->s.code.var_msgsize);
-//ncptl_log_write (logstate, 1, "1/2 RTT (usecs)", NCPTL_FUNC_MEDIAN, 0.0, ((double)var_elapsed_usecs)/(2.0));
-starttime += ncptl_time() - stop_elapsed_usecs;
-}
-break;
-
-default:
- /* The C code generation module must be broken. */
-ncptl_fatal ("Internal error: unknown EV_CODE block %" NICS, thisev->s.code.number);
-break;
-}
-break;
-
-default:
- /* The c_generic backend or the c_mpi backend must be broken. */
-ncptl_fatal ("Internal error: unknown event type %d", thisev->type);
-break;
-}
-}
-}
-
-/* Finish up cleanly and return a status code. */
-static int conc_finalize (void)
-{
-int exitcode = 0;   /* Program exit code (to pass to exit()) */
-
- /* Declarations specific to the c_mpi backend */
-//int mpiresult;   /* Return code from MPI_Finalize() */
-//char log_key_str[128];   /* String representing the range of valid MPI tags */
-//
-// /* Finalization code specific to the c_mpi backend */
-//log_key_str[0] = '\0';
-//if (conc_mcast_tallies[CONC_MCAST_MPI_BCAST] > 0) {
-//char onefuncstr[50];
-//sprintf (onefuncstr, "%sCODES_MPI_Bcast()*%" NICS,
-//log_key_str[0] == '\0' ? "" : " ", conc_mcast_tallies[CONC_MCAST_MPI_BCAST]);
-//strcat (log_key_str, onefuncstr);
-//}
-//if (conc_mcast_tallies[CONC_MCAST_MPI_ALLTOALL] > 0) {
-//char onefuncstr[50];
-//sprintf (onefuncstr, "%sCODES_MPI_Alltoall()*%" NICS,
-//log_key_str[0] == '\0' ? "" : " ", conc_mcast_tallies[CONC_MCAST_MPI_ALLTOALL]);
-//strcat (log_key_str, onefuncstr);
-//}
-//if (conc_mcast_tallies[CONC_MCAST_MPI_ALLTOALLV] > 0) {
-//char onefuncstr[50];
-//sprintf (onefuncstr, "%sCODES_MPI_Alltoallv()*%" NICS,
-//log_key_str[0] == '\0' ? "" : " ", conc_mcast_tallies[CONC_MCAST_MPI_ALLTOALLV]);
-//strcat (log_key_str, onefuncstr);
-//}
-//if (log_key_str[0] != '\0')
-//ncptl_log_add_comment ("Multicast functions used (statically)", log_key_str);
-//
-// /* Write a standard epilogue to the log file. */
-////ncptl_log_commit_data (logstate);
-////ncptl_log_write_epilogue (logstate);
-//ncptl_log_close (logstate);
-
- /* Inform the run-time library that it's no longer needed. */
-//ncptl_queue_empty (eventqueue);
-//ncptl_free (eventqueue);
-//ncptl_finalize();
-
- /* Finalization code specific to the c_mpi backend */
-CODES_MPI_Finalize();
-mpi_is_running = 0;
-exitcode = 0;
-
- /* Return an exit status code. */
-return exitcode;
-}
-
-/*************************************************************************/
-/***************************** MAIN ROUTINE ******************************/
-/*************************************************************************/
-
-/* Program execution starts here. */
-static int latencyall_main (int argc, char *argv[])
-{
- /* Declare variables needed by all C-based backends. */
-CONC_EVENT * eventlist;   /* List of events to execute */
-ncptl_int numevents;   /* Number of entries in eventlist[] */
-
- /* ----- Initialization ----- */
-conc_initialize (argc, argv);
-eventlist = (CONC_EVENT *) ncptl_queue_contents (eventqueue, 0);
-numevents = ncptl_queue_length (eventqueue);
-sendrequests = (MPI_Request *) ncptl_queue_contents (sendreqQ, 0);
-sendstatuses = (MPI_Status *) ncptl_queue_contents (sendstatQ, 0);
-recvrequests = (MPI_Request *) ncptl_queue_contents (recvreqQ, 0);
-recvstatuses = (MPI_Status *) ncptl_queue_contents (recvstatQ, 0);
-starttime = ncptl_time();
-
- /* ----- Event-list processing ----- */
-conc_process_events (eventlist, 0, numevents-1, 1);
-
- /* ----- Finalization ----- */
-return conc_finalize();
-}
-
-/* fill in function pointers for this method */
-struct codes_conceptual_bench latencyall_bench = 
-{
-.program_name = "latencyall",
-.conceptual_main = latencyall_main,
-};

From 1d34e479806ceef47086fe0ebdbb19ff7d9f78c8 Mon Sep 17 00:00:00 2001
From: Xin Wang <xwang925@gmail.com>
Date: Tue, 22 Jan 2019 15:19:25 -0600
Subject: [PATCH 15/15] remove conceptual benchmark examples

---
 Makefile.am                     |  5 -----
 src/workload/codes-conc-addon.c | 10 ----------
 2 files changed, 15 deletions(-)

diff --git a/Makefile.am b/Makefile.am
index 9d4d28db..51accd3b 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -71,11 +71,6 @@ if USE_CONC
 src_libcodes_la_SOURCES += src/workload/methods/codes-conc-online-comm-wrkld.C
 AM_CPPFLAGS += ${CONCEPTUAL_CFLAGS} -DUSE_CONC=1
 LDADD += ${CONCEPTUAL_LIBS}
-src_libcodes_la_SOURCES += src/workload/conceputal-skeleton-apps/conc-bisect.c
-src_libcodes_la_SOURCES += src/workload/conceputal-skeleton-apps/conc-cosmoflow.c
-src_libcodes_la_SOURCES += src/workload/conceputal-skeleton-apps/conc-hotpotato.c
-src_libcodes_la_SOURCES += src/workload/conceputal-skeleton-apps/conc-latencyall.c
-src_libcodes_la_SOURCES += src/workload/conceputal-skeleton-apps/conc-latency.c
 endif
 endif
 
diff --git a/src/workload/codes-conc-addon.c b/src/workload/codes-conc-addon.c
index 6b1d2ae8..ef5843b2 100644
--- a/src/workload/codes-conc-addon.c
+++ b/src/workload/codes-conc-addon.c
@@ -10,11 +10,6 @@
 
 #ifdef USE_CONC
 /* list of available benchmarks begin */
-extern struct codes_conceptual_bench bisect_bench;
-extern struct codes_conceptual_bench cosmoflow_bench;
-extern struct codes_conceptual_bench hotpotato_bench;
-extern struct codes_conceptual_bench latencyall_bench;
-extern struct codes_conceptual_bench latency_bench;
 /* list of available benchmarks end */
 #endif
 
@@ -22,11 +17,6 @@ static struct codes_conceptual_bench const * bench_array_default[] =
 {
 #ifdef USE_CONC
     /* default benchmarks begin */
-    &bisect_bench,
-    &cosmoflow_bench,
-    &hotpotato_bench,
-    &latencyall_bench,
-    &latency_bench,
     /* default benchmarks end */
 #endif
     NULL