libteac.c revision e0e0c19eefceaf5d4ec40f9466b58a771f50e799
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync * -Why is elan3_fini dropping core? It's intermittent, and seems to
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync * depend on relative timing of multiple calls.
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync fprintf(stderr,"failed to reallocate %d bytes!\n",size); \
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync/* We need some defs to handle changes in structures between software
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync * revisions.
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync/* Capability shape is the same between KITE and pre-KITE */
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync/* software level is PRE_KITE */
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync/* software level is KITE */
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync/* software level is EAGLE */
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync/* In the future we will use the bitwise AND of the rail masks */
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync#define HOST_TABLE_FILENAME "/usr/users/8/welling/elanstuff/teac/teac_host_map.t"
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsyncstatic host_t* hosts= NULL; /* this one ends up sorted by host name */
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsyncstatic int* hostsIndex= NULL; /* this is sorted by node ID */
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsyncstatic int nodeCount= 0;
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync if (!(hosts= (host_t*)crAlloc(INITIAL_HOST_TABLE_SIZE*sizeof(host_t)))) {
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync fprintf(stderr,"libteac: read_node_map: unable to allocate %d bytes!\n",
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync fprintf(stderr,"libteac: read_node_map: cannot open <%s> for reading!\n",
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync if (buf[0]=='#' || buf[0]=='\n' || buf[0]=='\0') continue;
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync /* We need to grow the table */
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync fprintf(stderr,"libteac: read_node_map: Bad machine name at line %d of %s!\n",
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync fprintf(stderr,"libteac: read_node_map: bad integer string at line %d of %s!\n",
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync fprintf(stderr,"libteac: read_node_map: bad integer %s at %s line %d!\n",
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync fprintf(stderr,"libteac: read_node_map: bad integer string at line %d of %s!\n",
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync fprintf(stderr,"libteac: read_node_map: bad integer %s at %s line %d!\n",
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync fprintf(stderr,"libteac: read_node_map: bad hex value at line %d of %s!\n",
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync hosts[i].sdramAddrBase= (sdramaddr_t)strtol(string, (char**)NULL, 0);
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync fprintf(stderr,"libteac: read_node_map: bad hex address %s at %s line %d!\n",
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync fprintf(stderr,"libteac: read_node_map: bad hex value at line %d of %s!\n",
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync hosts[i].elanAddrBase= (E3_Addr)strtol(string, (char**)NULL, 0);
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync fprintf(stderr,"libteac: read_node_map: bad hex address %s at %s line %d!\n",
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync iLine, i, hosts[i].name, hosts[i].railMask, hosts[i].id,
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync fprintf(stderr,"libteac: read_node_map: error reading <%s>!\n",
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsyncstatic int hostnameCompare(const void* h1, const void* h2)
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync return strcmp(((host_t*)h1)->name, ((host_t*)h2)->name);
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsyncstatic int hostnameLookup(const void* h1, const void* h2)
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync return strcmp((const char*)h1, ((host_t*)h2)->name);
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync crDebug("Loading Quadrics network map from <%s>\n",HOST_TABLE_FILENAME);
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync /* Load information about Quadrics network */
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync "libteac: initialize_node_tables: no valid nodes in %s!\n",
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync * Build the offset tables. Offsets are looked up by node ID, so we
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync * have to avoid redundant IDs and order the tables correctly.
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync if (!(sdramAddrBase=(sdramaddr_t*)crAlloc(nodeRange*sizeof(sdramaddr_t)))) {
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync "libteac: initialize_node_tables: unable to allocate %d bytes!\n",
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync if (!(elanAddrBase=(E3_Addr*)crAlloc(nodeRange*sizeof(E3_Addr)))) {
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync "libteac: initialize_node_tables: unable to allocate %d bytes!\n",
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync for (i=0; i<nodeRange; i++) {
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync for (i=0; i<nodeCount; i++) {
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync sdramAddrBase[hosts[i].id]= hosts[i].sdramAddrBase;
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync /* Sort the host table alphabetically by host name for faster lookup */
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync qsort( hosts, nodeCount, sizeof(host_t), hostnameCompare );
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync /* Build an ordered index into the hosts table */
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync if (!(hostsIndex= (int*)crAlloc(nodeRange*sizeof(int)))) {
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync fprintf(stderr,"libteac: read_node_map: unable to allocate %d bytes!\n",
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync nodeCount*sizeof(int));
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync for (i=0; i<nodeCount; i++) hostsIndex[hosts[i].id]= i;
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync if (!nodeTablesInitialized) initialize_node_tables();
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync while (i<sizeof(buf)-1 && hn[i] && hn[i]!='.') { buf[i]= hn[i]; i++; };
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync h=(host_t*)bsearch(hn, hosts, nodeCount, sizeof(host_t), hostnameLookup);
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync hn,h->name,h->id,(int)h->sdramAddrBase,(int)h->elanAddrBase);
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync else fprintf(stderr,"Lookup <%s> returned NULL!\n",hn);
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync if (h) return h->id;
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync else return -1;
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync * A version of address translation with some safety checks
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsyncstatic E3_Addr teac_main2elan( ELAN3_CTX *ctx, void* main_addr )
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync crDebug("mapping %0lx -> %d; addressable %d\n",
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync main_addr,result,elan3_addressable(ctx,main_addr,64));
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync fprintf(stderr,"Address translation error: %0x has no elan equivalent\n",
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsyncTcomm *teac_Init(char *lh, char *hh, int lctx, int hctx, int myrank,
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync if (!nodeTablesInitialized) initialize_node_tables();
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync fprintf(stderr,"teac_Init: unable to allocate %d bytes!\n",
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync for (i=0; i<NUM_SEND_BUFFERS; i++) result->sbuf_pull_event[i] = 0;
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync for (i=0; i<NUM_SEND_BUFFERS; i++) result->sbuf_ready[i]= NULL;
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync for (i=0; i<NUM_SEND_BUFFERS; i++) result->sendWrappers[i]= NULL;
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync /* Initialize the UserKey to the given value */
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync crMemcpy((void*)&(cap->cap_userkey.key_values),key,TEAC_KEY_SIZE);
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync ELAN_CAP_TYPE_BLOCK | ELAN_CAP_TYPE_NO_BITMAP | ELAN_CAP_TYPE_MULTI_RAIL;
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync /* Initialize the UserKey to the given value */
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync crMemcpy((void*)&(cap->UserKey),key,TEAC_KEY_SIZE);
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync cap->Entries = (hctx - lctx + 1) * (cap->HighNode - cap->LowNode + 1);
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync ELAN_CAP_TYPE_BLOCK | ELAN_CAP_TYPE_NO_BITMAP | ELAN_CAP_TYPE_MULTI_RAIL;
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync if ((result->ctx = elan3_init( 0, EADDR_ALLOC_MAIN, ALLOC_MAIN_SIZE,
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync fprintf(stderr, "teac_Init: elan3_init returned NULL context\n");
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync if ((control = elan3_control_open (RAIL)) != NULL) {
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync crDebug("elan3_create failed with <%s>, but that's OK!\n",
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync crDebug("elan3_create failed with <%s>, but that's OK!\n",
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync /* I don't think we have to do anything here! */
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync crDebug("Position mode %d, NodeID %d, NumNodes %d, NumLevels %d\n",
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync position.pos_mode,position.pos_nodeid, position.pos_nodes,
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync crDebug("WARNING: position mode is not %d!\n",ELAN_POS_MODE_SWITCHED);
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync crDebug("NodeId: %d, NumNodes: %d, NumLevels: %d, NodeLevel: %d\n",
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync crDebug("NodeId: %d, NumLevels: %d, NodeLevel: %d\n",
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync elan_capability_string(&(result->cap),junkString));
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync crDebug("railmask is %d\n",result->cap.cap_railmask);
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync crDebug("bitmap is %x\n",result->cap.cap_bitmap[0]);
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync elan3_capability_string(&(result->cap),junkString));
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync /* Reality check. */
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync if ((here= crStrchr(buf,'.')) != NULL) *here= '\0';
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync "teac_Init: Expected Quadrics port id %d does not match real value %d!\n",
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync "teac_Init: Expected Quadrics port id %d does not match real value %d!\n",
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync "teac_Init: Expected Quadrics port id %d does not match real value %d!\n",
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync fprintf(stderr, "teac_Init: elan3_attach failed\n");
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync fprintf(stderr, "teac_Init: elan3_addvp failed\n");
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync fprintf(stderr, "teac_Init: elan3_set_required_mappings failed\n");
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync (sdramaddr_t**)crAlloc( TEAC_CAP_ENTRIES(cap)*sizeof(sdramaddr_t*) ))) {
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync fprintf(stderr,"teac_Init: unable to allocate %d bytes!\n",
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync (sdramaddr_t*)crAlloc( TEAC_CAP_ENTRIES(cap)*NUM_SEND_BUFFERS
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync * sizeof(sdramaddr_t) ))) {
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync fprintf(stderr,"teac_Init: unable to allocate %d bytes!\n",
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync TEAC_CAP_ENTRIES(cap)*NUM_SEND_BUFFERS*sizeof(sdramaddr_t));
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync TEAC_CAP_ENTRIES(cap)*NUM_SEND_BUFFERS*sizeof(E3_BlockCopyEvent)))) {
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync perror("teac_Init: elan3_allocElan failed for r_event block");
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync result->r_event[j]= result->r_event[0] + j*NUM_SEND_BUFFERS;
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync result->r_event[0][0]+j*NUM_SEND_BUFFERS*sizeof(E3_BlockCopyEvent);
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync result->r_event[j][0] + i*sizeof(E3_BlockCopyEvent);
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync crDebug("r_event[0][0] is %x\n",(int)result->r_event[0][0]);
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync (volatile E3_uint32**)crAlloc( TEAC_CAP_ENTRIES(cap)*sizeof(E3_uint32*) ))) {
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync fprintf(stderr,"teac_Init: unable to allocate %d bytes!\n",
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync TEAC_CAP_ENTRIES(cap)*NUM_SEND_BUFFERS*sizeof(E3_uint32)))) {
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync perror("teac_Init: elan3_allocMain failed for m_rcv block");
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync result->m_rcv[i]= result->m_rcv[0] + i*NUM_SEND_BUFFERS;
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync (long)teac_main2elan(result->ctx,(void*)(result->m_rcv[0])));
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync if (!(result->mbuff= (teacMsg**)crAlloc( TEAC_CAP_ENTRIES(cap)*sizeof(teacMsg*) ))) {
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync fprintf(stderr,"teac_Init: unable to allocate %d bytes!\n",
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync TEAC_CAP_ENTRIES(cap)*NUM_SEND_BUFFERS*sizeof(teacMsg)))) {
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync perror("teac_Init: elan3_allocMain failed for mbuff block");
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync result->mbuff[i]= result->mbuff[0] + i*NUM_SEND_BUFFERS;
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync (long)teac_main2elan(result->ctx,result->mbuff[0]));
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync perror("teac_Init: elan3_allocMain failed for dma");
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync elan3_allocElan(result->ctx, E3_DMA_ALIGN, sizeof(E3_DMA)))) {
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync perror("teac_Init: elan3_allocElan failed for e_dma");
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync perror("teac_Init: elan3_allocElan failed for s_event");
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync perror("teac_Init: elan3_allocElan failed for sbuf_pull_event block");
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync result->sbuf_pull_event[0]+i*sizeof(E3_BlockCopyEvent);
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync perror("teac_Init: elan3_allocElan failed for rbuf_pull_event");
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync perror("teac_Init: elan3_allocMain failed for m_snd");
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync perror("teac_Init: elan3_allocMain failed for sbuf_ready block");
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync (SBuffer*)crAlloc(NUM_SEND_BUFFERS*sizeof(SBuffer)))) {
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync fprintf(stderr,"teac_Init: unable to allocate %d bytes!\n",
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync result->sendWrappers[i]= result->sendWrappers[0] + i;
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync perror("teac_Init: elan3_allocMain failed for rbuf_ready");
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync for (i=0; i<NUM_SEND_BUFFERS; i++) {
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync perror("teac_Init: elan3_allocMain failed for buffer block");
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync result->sendWrappers[i]->totSize= E_BUFFER_INITIAL_SIZE;
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync result->totalSendBufferBytesAllocated += result->sendWrappers[i]->totSize;
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync for (i=0; i<NUM_SEND_BUFFERS; i++) {
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync result->sbuf_pull_event[i], result->sbuf_ready[i]);
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync for (i=0; i<NUM_SEND_BUFFERS; i++) {
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync /* Get the message receive events ready to fire, in case something
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync * comes in before receive gets called.
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync for (i=0; i<NUM_SEND_BUFFERS; i++) {
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync elan3_primeevent(result->ctx, result->r_event[j][i], 1);
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync /* Fire the sbuffer free events, so that the buffers look free when
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync * the first call to send happens.
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync for (i=0; i<NUM_SEND_BUFFERS; i++) {
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync elan3_primeevent(result->ctx, result->sbuf_pull_event[i], 1);
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync elan3_setevent( result->ctx, result->sbuf_pull_event[i] );
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync /* And now we're ready to face the world. */
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync /* First we have to wait until all pending messages have been
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync * pulled (assuming they got initialized in the first place).
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync if ((tcomm->sbuf_pull_event[0] != 0) && (tcomm->sbuf_ready[0] != NULL)) {
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync for (i=0; i<NUM_SEND_BUFFERS; i++) {
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync elan3_waitevent_word(tcomm->ctx, tcomm->sbuf_pull_event[i],
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync crDebug("All TEAC messages have reported home!\n");
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync if (tcomm->e_dma != 0) elan3_freeElan(tcomm->ctx, tcomm->e_dma);
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync if (tcomm->s_event != 0) elan3_freeElan(tcomm->ctx, tcomm->s_event);
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync if (tcomm->r_event[0][0] != 0) elan3_freeElan(tcomm->ctx,
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync elan3_freeElan(tcomm->ctx, tcomm->sbuf_pull_event[0]);
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync for (i=0; i<NUM_SEND_BUFFERS; i++) {
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync elan3_free(tcomm->ctx, tcomm->sendWrappers[i]->buf);
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync elan3_freeElan(tcomm->ctx, tcomm->rbuf_pull_event);
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync if (tcomm->m_rcv[0] != NULL) elan3_free(tcomm->ctx,
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync elan3_free(tcomm->ctx, (E3_uint32*)tcomm->sbuf_ready[0]);
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync if (tcomm->rbuf_ready != NULL) elan3_free(tcomm->ctx,
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync if (tcomm->mbuff[0] != NULL) elan3_free(tcomm->ctx, tcomm->mbuff[0]);
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync /* elan3_detach and elan3_destroy seem to crash sometimes in
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync * these versions.
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsyncint teac_Select(Tcomm* tcomm, int *ids, int num_ids, int timeout)
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync while (timeout-- > 0) {
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync if ((id= teac_Poll(tcomm, ids, num_ids)) >= 0) return id;
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync for (j=0; j<num_ids; j++) {
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync for (i=0; i<NUM_SEND_BUFFERS; i++) {
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync if (elan3_pollevent_word(tcomm->ctx, &(tcomm->m_rcv[thisId][i]), 1)) {
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync for (i=0; i<NUM_SEND_BUFFERS; i++) {
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync if (elan3_pollevent_word(tcomm->ctx, tcomm->sbuf_ready[i], 1))
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsyncSBuffer* teac_getSendBuffer( Tcomm* tcomm, long size )
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync /* Find a free send buffer. We'll busy wait in this poll loop
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync * if necessary.
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync while (1) {
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync if (elan3_pollevent_word(tcomm->ctx, tcomm->sbuf_ready[i],
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync if (++i == NUM_SEND_BUFFERS) i= 0;
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync /* We will use this buffer! */
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync /* If the associated chunk of memory is smaller than that requested,
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync * replace it with something larger.
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync tcomm->totalSendBufferBytesAllocated -= tcomm->sendWrappers[i]->totSize;
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync elan3_free( tcomm->ctx, tcomm->sendWrappers[i]->buf );
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync perror("teac_getSendBuffer: failed to grow send buffer");
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsyncSBuffer* teac_getUnreadySendBuffer( Tcomm* tcomm, long size )
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync if (!(result= (SBuffer*)crAlloc(sizeof(SBuffer)))) {
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync fprintf(stderr,"libteac: read_node_map: unable to allocate %d bytes!\n",
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync /* Allocate some DMA-able memory */
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync if (!(result->buf= (char*)elan3_allocMain(tcomm->ctx, 8, size))) {
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync perror("teac_getUnreadySendBuffer: failed to allocate elan3 memory");
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync tcomm->totalSendBufferBytesAllocated += result->totSize;
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsyncSBuffer* teac_makeSendBufferReady( Tcomm* tcomm, SBuffer* buf )
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync /* If the input buffer is already ready, just return it */
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync if (buf->bufId >= 0 && buf->bufId<NUM_SEND_BUFFERS) return buf;
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync /* Find a free send buffer. We'll busy wait in this poll loop
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync * if necessary.
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync while (1) {
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync if (elan3_pollevent_word(tcomm->ctx, tcomm->sbuf_ready[i],
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync if (++i == NUM_SEND_BUFFERS) i= 0;
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync /* We will use this buffer! */
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync crDebug("Allocated message buffer %d in makeSendBufferReady\n",i);
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync /* Substitute the unready payload for the old payload */
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync tcomm->totalSendBufferBytesAllocated -= tcomm->sendWrappers[i]->totSize;
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync elan3_free( tcomm->ctx, tcomm->sendWrappers[i]->buf );
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsyncint teac_Send( Tcomm* tcomm, int* ids, int num_ids, SBuffer* buf, void *start )
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync /* Complain loudly if this is an unready buffer */
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync fprintf(stderr,"teac_Send: tried to send an unready buffer!\n");
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync /* Reality check: is this one of my buffers? */
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync if (buf->bufId<0 || buf->bufId>=NUM_SEND_BUFFERS) {
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync fprintf(stderr,"teac_Send: I don't know this buffer!\n");
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync /* Reality check: did they write too much into the message? */
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync fprintf(stderr,"teac_Send: message too large! (%ld vs %ld)\n",
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync msg->clr_event = elan3_sdram2elan(tcomm->ctx, tcomm->ctx->sdram,
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync /* Set up the parts of the DMA that are not specific to destination */
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync tcomm->dma->dma_type = E3_DMA_TYPE(DMA_BYTE,DMA_WRITE,DMA_NORMAL,0);
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync elan3_sdram2elan(tcomm->ctx,tcomm->ctx->sdram,tcomm->s_event);
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync tcomm->dma->dma_source = teac_main2elan(tcomm->ctx, msg);
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync elan3_primeevent(tcomm->ctx, tcomm->sbuf_pull_event[buf->bufId], num_ids);
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync /* Send this message off to each destination */
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync + sdramAddrBase[(ids[iDest]/NUM_SEND_BUFFERS) + tcomm->lhost]
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync - sdramAddrBase[(vp/NUM_SEND_BUFFERS) + tcomm->lhost];
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync tcomm->dma->dma_dest = teac_main2elan(tcomm->ctx, msg)
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync + elanAddrBase[(ids[iDest]/NUM_SEND_BUFFERS) + tcomm->lhost]
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync - elanAddrBase[(vp/NUM_SEND_BUFFERS) + tcomm->lhost];
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync elan3_putdma_main(tcomm->ctx, tcomm->dma, tcomm->e_dma);
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync elanAddrBase[(ids[iDest]/NUM_SEND_BUFFERS) + tcomm->lhost],
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync elanAddrBase[(vp/NUM_SEND_BUFFERS) + tcomm->lhost],
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync crDebug("Send msg %d in buffer %d to %d (list index %d)...",
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync /* poll until we get an incoming message from the given sender */
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync while (1) {
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync if (elan3_pollevent_word(tcomm->ctx, &(tcomm->m_rcv[id][iEvent]), 1))
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync /* This may not be the earliest event, however. */
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync for (i=0; i<NUM_SEND_BUFFERS; i++) {
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync (long)teac_main2elan(tcomm->ctx,(void*)(&tcomm->mbuff[id][i])));
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync || (tcomm->mbuff[id][i].msgnum < (E3_uint32)lowestMsgnum)) {
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync "teac_Recv: internal error: can't find message I just got!\n");
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync elan3_primeevent(tcomm->ctx, tcomm->r_event[id][iBuf],1);
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync /* Make some space to put the message when it arrives */
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync if (!(result= (RBuffer*)crAlloc(sizeof(RBuffer)))) {
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync fprintf(stderr,"teac_Recv: unable to allocate %d bytes!\n",
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync if (!(result->buf= (void*)elan3_allocMain(tcomm->ctx, 8,E_BUFFER_SIZE))) {
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync perror("teac_Recv: elan3_allocMain failed for buffer");
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync if (!(result->buf= (void*)elan3_allocMain(tcomm->ctx, 8,
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync perror("teac_Recv: elan3_allocMain failed for buffer");
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync tcomm->totalRecvBufferBytesAllocated+= result->totSize;
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync result->senderMsgnum= tcomm->mbuff[id][iBuf].msgnum;
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync /* Set up the DMA to pull the message */
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync tcomm->dma->dma_type = E3_DMA_TYPE(DMA_BYTE,DMA_READ,DMA_NORMAL,0);
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync tcomm->dma->dma_size = tcomm->mbuff[id][iBuf].size;
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync tcomm->dma->dma_source = tcomm->mbuff[id][iBuf].mptr;
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync tcomm->dma->dma_dest = teac_main2elan(tcomm->ctx,result->buf);
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync elan3_remote_cookie(tcomm->ctx, vp, tcomm->mbuff[id][iBuf].host);
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync elan3_local_cookie(tcomm->ctx, vp, tcomm->mbuff[id][iBuf].host);
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync tcomm->dma->dma_srcEvent = tcomm->mbuff[id][iBuf].clr_event;
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync elan3_primeevent(tcomm->ctx, tcomm->rbuf_pull_event,1);
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync elan3_getdma_main(tcomm->ctx, tcomm->dma, tcomm->e_dma);
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync tcomm->totalRecvBufferBytesAllocated -= buf->totSize;
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsyncchar* teac_getTcommString(Tcomm *c, char* buf, int buflen)
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync "<vp= %d, host range %d-%d, ctxnum range %d-%d, msg %d>",
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync c->vp,c->lhost,c->hhost,c->lctx,c->hctx,c->msgnum);
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsyncchar* teac_getConnString(Tcomm *c, int id, char* buf, int buflen)
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync crDebug("getConnString: lookup id %d -> %d %d -> %s\n",id,rel_rank,node,buf);
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsyncint teac_getConnId(Tcomm *c, const char* host, int rank)
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync host, rank, c->lhost, node, (4*(node - c->lhost) + (rank%4)));
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsyncint teac_getHostInfo(Tcomm *c, char* host, const int hostLength,
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync if ((here= crStrchr(host,'.')) != NULL) *here= '\0';
e0e0c19eefceaf5d4ec40f9466b58a771f50e799vboxsync *elanBaseAddr= (long)teac_main2elan(c->ctx,(void*)c->m_rcv[0]);