diff options
author | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-05-06 03:01:46 +0000 |
---|---|---|
committer | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-05-06 03:01:46 +0000 |
commit | f8fe689a81f906d1b91bb3220acde2a4ecb14c5b (patch) | |
tree | 26484e9d7e2c67806c2d1760196ff01aaa858e8c /src/VBox/GuestHost/OpenGL/util/libteac.c | |
parent | Initial commit. (diff) | |
download | virtualbox-f8fe689a81f906d1b91bb3220acde2a4ecb14c5b.tar.xz virtualbox-f8fe689a81f906d1b91bb3220acde2a4ecb14c5b.zip |
Adding upstream version 6.0.4-dfsg.upstream/6.0.4-dfsgupstream
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'src/VBox/GuestHost/OpenGL/util/libteac.c')
-rw-r--r-- | src/VBox/GuestHost/OpenGL/util/libteac.c | 1213 |
1 files changed, 1213 insertions, 0 deletions
diff --git a/src/VBox/GuestHost/OpenGL/util/libteac.c b/src/VBox/GuestHost/OpenGL/util/libteac.c new file mode 100644 index 00000000..eb349809 --- /dev/null +++ b/src/VBox/GuestHost/OpenGL/util/libteac.c @@ -0,0 +1,1213 @@ +#include <stdio.h> +#include <stdlib.h> +#include <sys/fcntl.h> +#include <string.h> +#include <time.h> + +#include "teac.h" + +/* Notes- + * -Why is elan3_fini dropping core? It's intermittent, and seems to + * depend on relative timing of multiple calls. + */ + +#ifdef CHROMIUM +#include <cr_string.h> +#include <cr_mem.h> +#include <cr_error.h> +#else +#define crAlloc(sz) malloc(sz) +#define crStrncpy(out,in,sz) strncpy(out,in,sz) +#define crMemcpy(out,in,bytes) memcpy(out,in,bytes) +#define crStrchr(instr,inchr) strchr(instr,inchr) +#define crFree(ptr) free(ptr) +#define crRealloc( pp, size ) { \ + if (!(*pp=realloc(*pp,size))) { \ + fprintf(stderr,"failed to reallocate %d bytes!\n",size); \ + abort(); \ + } \ +} +#include <stdarg.h> +static void crDebug( const char* fmt, ... ) { + va_list ap; + va_start(ap,fmt); + vfprintf(stderr, fmt, ap); + va_end(ap); +} +#endif + +#define EADDR_ALLOC_ELAN 0x200000 +#define ALLOC_ELAN_SIZE 0x200000 +#define EADDR_ALLOC_MAIN 0x400000 +#define ALLOC_MAIN_SIZE 0x2000000 + +/* We need some defs to handle changes in structures between software + * revisions. + */ +/* Capability shape is the same between KITE and pre-KITE */ +#ifdef ELAN_PRE_KITE + +/* software level is PRE_KITE */ +#define TEAC_DEVINFO_TYPE 0 +#define TEAC_CAP_TYPE 0 + +#else +#ifdef ELAN_PRE_EAGLE + +/* software level is KITE */ + +#ifdef LINUX +#define TEAC_DEVINFO_TYPE 1 +#define TEAC_CREATE_TYPE 1 +#else +#define TEAC_DEVINFO_TYPE 0 +#define TEAC_CREATE_TYPE 0 +#endif +#define TEAC_CAP_TYPE 1 + +#else +/* software level is EAGLE */ +#define TEAC_DEVINFO_TYPE 2 +#define TEAC_CAP_TYPE 2 +#define TEAC_CREATE_TYPE 2 + +#endif +#endif + +#if (TEAC_CAP_TYPE == 2) +#define TEAC_CAP_ENTRIES(cap) ELAN_CAP_ENTRIES(cap) +#else +#define TEAC_CAP_ENTRIES(cap) (cap->Entries) +#endif + +/* In the future we will use the bitwise AND of the rail masks */ +#define RAIL 0 + +#ifndef HOST_TABLE_FILENAME +#define HOST_TABLE_FILENAME "/usr/users/8/welling/elanstuff/teac/teac_host_map.t" +#endif + +#define INITIAL_HOST_TABLE_SIZE 256 + +static host_t* hosts= NULL; /* this one ends up sorted by host name */ +static int* hostsIndex= NULL; /* this is sorted by node ID */ +static sdramaddr_t* sdramAddrBase= NULL; +static E3_Addr* elanAddrBase= NULL; + +static int nodeTablesInitialized= 0; +static int nodeTableSize= 0; +static int nodeCount= 0; + +static int read_node_map() +{ + FILE* f= NULL; + int i; + int iLine; + char buf[256]; + + if (hosts) crFree(hosts); + if (!(hosts= (host_t*)crAlloc(INITIAL_HOST_TABLE_SIZE*sizeof(host_t)))) { + fprintf(stderr,"libteac: read_node_map: unable to allocate %d bytes!\n", + INITIAL_HOST_TABLE_SIZE*sizeof(host_t)); + abort(); + } + nodeTableSize= INITIAL_HOST_TABLE_SIZE; + + if (!(f=fopen(HOST_TABLE_FILENAME,"r"))) { + fprintf(stderr,"libteac: read_node_map: cannot open <%s> for reading!\n", + HOST_TABLE_FILENAME); + abort(); + } + + i= 0; + iLine= 0; + buf[sizeof(buf)-1]= '\0'; + while (!feof(f) && !ferror(f)) { + char* tmp; + char* string; + if (!fgets(buf, sizeof(buf)-1, f)) break; + iLine++; + if (buf[0]=='#' || buf[0]=='\n' || buf[0]=='\0') continue; + if (i>=nodeTableSize) { + /* We need to grow the table */ + int newSize= 2*nodeTableSize; + crRealloc((void**)&hosts,newSize*sizeof(host_t)); + nodeTableSize= newSize; + } + if (!(string=strtok_r(buf," ,;\t\n",&tmp))) { + fprintf(stderr,"libteac: read_node_map: Bad machine name at line %d of %s!\n", + iLine, HOST_TABLE_FILENAME); + abort(); + } + if (strlen(string)==0) continue; /* blank line */ + hosts[i].name= strdup(string); + + if (!(string=strtok_r(NULL," ,;\t\n",&tmp))) { + fprintf(stderr,"libteac: read_node_map: bad integer string at line %d of %s!\n", + iLine, HOST_TABLE_FILENAME); + abort(); + } + errno= 0; + hosts[i].railMask= atoi(string); + if (errno != 0) { + fprintf(stderr,"libteac: read_node_map: bad integer %s at %s line %d!\n", + string, HOST_TABLE_FILENAME, iLine); + abort(); + } + + if (!(string=strtok_r(NULL," ,;\t\n",&tmp))) { + fprintf(stderr,"libteac: read_node_map: bad integer string at line %d of %s!\n", + iLine, HOST_TABLE_FILENAME); + abort(); + } + errno= 0; + hosts[i].id= atoi(string); + if (errno != 0) { + fprintf(stderr,"libteac: read_node_map: bad integer %s at %s line %d!\n", + string, HOST_TABLE_FILENAME, iLine); + abort(); + } + + if (!(string=strtok_r(NULL," ,;\t\n",&tmp))) { + fprintf(stderr,"libteac: read_node_map: bad hex value at line %d of %s!\n", + iLine, HOST_TABLE_FILENAME); + abort(); + } + errno= 0; + hosts[i].sdramAddrBase= (sdramaddr_t)strtol(string, (char**)NULL, 0); + if (errno != 0) { + fprintf(stderr,"libteac: read_node_map: bad hex address %s at %s line %d!\n", + string, HOST_TABLE_FILENAME, iLine); + abort(); + } + + if (!(string=strtok_r(NULL," ,;\t\n",&tmp))) { + fprintf(stderr,"libteac: read_node_map: bad hex value at line %d of %s!\n", + iLine, HOST_TABLE_FILENAME); + abort(); + } + errno= 0; + hosts[i].elanAddrBase= (E3_Addr)strtol(string, (char**)NULL, 0); + if (errno != 0) { + fprintf(stderr,"libteac: read_node_map: bad hex address %s at %s line %d!\n", + string, HOST_TABLE_FILENAME, iLine); + abort(); + } + +#if 0 + crDebug("line %d: %d: got <%s> %d %d 0x%x 0x%x\n", + iLine, i, hosts[i].name, hosts[i].railMask, hosts[i].id, + hosts[i].sdramAddrBase, hosts[i].elanAddrBase); +#endif + + i++; + } + if (ferror(f)) { + perror("Error reading host table"); + fprintf(stderr,"libteac: read_node_map: error reading <%s>!\n", + HOST_TABLE_FILENAME); + abort(); + } + + (void)fclose(f); + return i; +} + +static int hostnameCompare(const void* h1, const void* h2) +{ + return strcmp(((host_t*)h1)->name, ((host_t*)h2)->name); +} + +static int hostnameLookup(const void* h1, const void* h2) +{ + return strcmp((const char*)h1, ((host_t*)h2)->name); +} + +static void initialize_node_tables() +{ + if (!nodeTablesInitialized) { + int nodeMin; + int nodeMax; + int nodeRange; + int i; + + crDebug("Loading Quadrics network map from <%s>\n",HOST_TABLE_FILENAME); + + /* Load information about Quadrics network */ + nodeCount= read_node_map(); + if (nodeCount==0) { + fprintf(stderr, + "libteac: initialize_node_tables: no valid nodes in %s!\n", + HOST_TABLE_FILENAME); + abort(); + } + + /* + * Build the offset tables. Offsets are looked up by node ID, so we + * have to avoid redundant IDs and order the tables correctly. + */ + nodeMin= nodeMax= hosts[0].id; + for (i=1; i<nodeCount; i++) { + if (hosts[i].id<nodeMin) nodeMin= hosts[i].id; + if (hosts[i].id>nodeMax) nodeMax= hosts[i].id; + } + nodeRange= (nodeMax-nodeMin) + 1; + if (!(sdramAddrBase=(sdramaddr_t*)crAlloc(nodeRange*sizeof(sdramaddr_t)))) { + fprintf(stderr, + "libteac: initialize_node_tables: unable to allocate %d bytes!\n", + nodeRange*sizeof(sdramaddr_t)); + abort(); + } + if (!(elanAddrBase=(E3_Addr*)crAlloc(nodeRange*sizeof(E3_Addr)))) { + fprintf(stderr, + "libteac: initialize_node_tables: unable to allocate %d bytes!\n", + nodeRange*sizeof(E3_Addr)); + abort(); + } + for (i=0; i<nodeRange; i++) { + sdramAddrBase[i]= (sdramaddr_t)0; + elanAddrBase[i]= (E3_Addr)0; + } + for (i=0; i<nodeCount; i++) { + sdramAddrBase[hosts[i].id]= hosts[i].sdramAddrBase; + elanAddrBase[hosts[i].id]= hosts[i].elanAddrBase; + } + + /* Sort the host table alphabetically by host name for faster lookup */ + qsort( hosts, nodeCount, sizeof(host_t), hostnameCompare ); + + /* Build an ordered index into the hosts table */ + if (hostsIndex) crFree(hostsIndex); + if (!(hostsIndex= (int*)crAlloc(nodeRange*sizeof(int)))) { + fprintf(stderr,"libteac: read_node_map: unable to allocate %d bytes!\n", + nodeCount*sizeof(int)); + abort(); + } + for (i=0; i<nodeRange; i++) hostsIndex[i]= 0; + for (i=0; i<nodeCount; i++) hostsIndex[hosts[i].id]= i; + + nodeTablesInitialized= 1; + } +} + +static int trans_host(const char *hn) { + host_t* h= NULL; + + if (!nodeTablesInitialized) initialize_node_tables(); + +#if 0 + char buf[128]; + int i; + i= 0; + while (i<sizeof(buf)-1 && hn[i] && hn[i]!='.') { buf[i]= hn[i]; i++; }; + buf[i]= '\0'; +#endif + + h=(host_t*)bsearch(hn, hosts, nodeCount, sizeof(host_t), hostnameLookup); +#if 0 + if (h) { + crDebug("Lookup <%s> got <%s> <%d> <%x> <%x>\n", + hn,h->name,h->id,(int)h->sdramAddrBase,(int)h->elanAddrBase); + } + else fprintf(stderr,"Lookup <%s> returned NULL!\n",hn); +#endif + if (h) return h->id; + else return -1; +} + +/* + * A version of address translation with some safety checks + */ +static E3_Addr teac_main2elan( ELAN3_CTX *ctx, void* main_addr ) +{ + E3_Addr result= elan3_main2elan(ctx,main_addr); + /* + crDebug("mapping %0lx -> %d; addressable %d\n", + main_addr,result,elan3_addressable(ctx,main_addr,64)); + */ + if (result==ELAN_BAD_ADDR) { + fprintf(stderr,"Address translation error: %0x has no elan equivalent\n", + (int)main_addr); + abort(); + } + return result; +} + +Tcomm *teac_Init(char *lh, char *hh, int lctx, int hctx, int myrank, + const unsigned char key[TEAC_KEY_SIZE]) +{ +#if (TEAC_DEVINFO_TYPE == 2) + ELAN_DEVINFO info; + ELAN_POSITION position; +#else + ELAN3_DEVINFO info; +#endif + ELAN_CAPABILITY *cap; + Tcomm* result= NULL; + int i; + int j; + int a; + int b; + char buf[256]; + char* here; +#if ( TEAC_CREATE_TYPE == 0 ) + void* control; +#endif + + if (!nodeTablesInitialized) initialize_node_tables(); + + if (!(result= (Tcomm*)crAlloc(sizeof(Tcomm)))) { + fprintf(stderr,"teac_Init: unable to allocate %d bytes!\n", + sizeof(Tcomm)); + return(NULL); + } + result->ctx = NULL; + result->dma = NULL; + result->e_dma = 0; + result->s_event = 0; + result->r_event = NULL; + for (i=0; i<NUM_SEND_BUFFERS; i++) result->sbuf_pull_event[i] = 0; + result->rbuf_pull_event = 0; + result->m_snd = NULL; + result->m_rcv = NULL; + for (i=0; i<NUM_SEND_BUFFERS; i++) result->sbuf_ready[i]= NULL; + result->rbuf_ready = NULL; + result->mbuff= NULL; + for (i=0; i<NUM_SEND_BUFFERS; i++) result->sendWrappers[i]= NULL; + result->vp = -1; + result->hhost = result->lhost = -1; + result->hctx = result->lctx = -1; + result->msgnum = 0; + result->poll_shift = 0; + result->totalSendBufferBytesAllocated= 0; + result->totalRecvBufferBytesAllocated= 0; + + a = trans_host(lh); + b = trans_host(hh); + result->lhost = (a < b)? a : b; + result->hhost = (a > b) ? a : b; + + result->lctx = (lctx<hctx) ? lctx : hctx; + result->hctx = (hctx>lctx) ? hctx : lctx; + + cap = &(result->cap); +#if (TEAC_CAP_TYPE == 2) + elan_nullcap(cap); + /* Initialize the UserKey to the given value */ + crMemcpy((void*)&(cap->cap_userkey.key_values),key,TEAC_KEY_SIZE); + cap->cap_lowcontext = lctx; + cap->cap_mycontext = lctx + (myrank%4); + cap->cap_highcontext = hctx; + cap->cap_lownode = result->lhost; + cap->cap_highnode = result->hhost; + cap->cap_type = + ELAN_CAP_TYPE_BLOCK | ELAN_CAP_TYPE_NO_BITMAP | ELAN_CAP_TYPE_MULTI_RAIL; +#else + elan3_nullcap(cap); + /* Initialize the UserKey to the given value */ + crMemcpy((void*)&(cap->UserKey),key,TEAC_KEY_SIZE); + cap->LowContext = lctx; + cap->MyContext = lctx + (myrank%4); + cap->HighContext = hctx; + cap->LowNode = result->lhost; + cap->HighNode = result->hhost; + cap->Entries = (hctx - lctx + 1) * (cap->HighNode - cap->LowNode + 1); + cap->Type = + ELAN_CAP_TYPE_BLOCK | ELAN_CAP_TYPE_NO_BITMAP | ELAN_CAP_TYPE_MULTI_RAIL; +#endif + + if ((result->ctx = elan3_init( 0, EADDR_ALLOC_MAIN, ALLOC_MAIN_SIZE, + EADDR_ALLOC_ELAN, ALLOC_ELAN_SIZE)) + == NULL) { + fprintf(stderr, "teac_Init: elan3_init returned NULL context\n"); + return(NULL); + } + elan3_block_inputter (result->ctx, 1); + +#if ( TEAC_CREATE_TYPE == 0 ) + if ((control = elan3_control_open (RAIL)) != NULL) { + if (elan3_create(control, &(result->cap))) { + crDebug("elan3_create failed with <%s>, but that's OK!\n", + strerror(errno)); + errno= 0; + } + } + else { + perror("elan3_control_open failed"); + teac_Close(result); + return NULL; + } +#elif ( TEAC_CREATE_TYPE == 1 ) + if (elan3_create(result->ctx, &(result->cap))) { + crDebug("elan3_create failed with <%s>, but that's OK!\n", + strerror(errno)); + errno= 0; + } +#else + /* I don't think we have to do anything here! */ +#endif + +#if (TEAC_DEVINFO_TYPE == 2) + (void)elan3_get_devinfo(result->ctx, &info); + (void)elan3_get_position(result->ctx, &position); + crDebug("Position mode %d, NodeID %d, NumNodes %d, NumLevels %d\n", + position.pos_mode,position.pos_nodeid, position.pos_nodes, + position.pos_levels); + if (position.pos_mode != ELAN_POS_MODE_SWITCHED) + crDebug("WARNING: position mode is not %d!\n",ELAN_POS_MODE_SWITCHED); + crDebug("Rail %d\n",info.dev_rail); +#elif (TEAC_DEVINFO_TYPE == 1) + elan3_devinfo(0, &info); + crDebug("NodeId: %d, NumNodes: %d, NumLevels: %d, NodeLevel: %d\n", + info.Position.NodeId, info.Position.NumNodes, + info.Position.NumLevels, info.Position.NodeLevel); +#else + elan3_devinfo(0, &info); + crDebug("NodeId: %d, NumLevels: %d, NodeLevel: %d\n", + info.NodeId, info.NumLevels, info.NodeLevel); +#endif + +#if 0 + { + static char junkString[256]; +#if (TEAC_CAP_TYPE == 2) + crDebug("Capability: <%s>\n", + elan_capability_string(&(result->cap),junkString)); + crDebug("railmask is %d\n",result->cap.cap_railmask); + crDebug("bitmap is %x\n",result->cap.cap_bitmap[0]); +#else + crDebug("Capability: <%s>\n", + elan3_capability_string(&(result->cap),junkString)); + crDebug("railmask is %d\n",result->cap.RailMask); + crDebug("bitmap is %x\n",result->cap.Bitmap[0]); +#endif + } +#endif + + /* Reality check. */ + if (gethostname(buf,sizeof(buf)-1)) { + perror("Can't get my own host name"); + teac_Close(result); + return NULL; + } + if ((here= crStrchr(buf,'.')) != NULL) *here= '\0'; + +#if (TEAC_DEVINFO_TYPE == 2) + if (trans_host(buf) != (int)position.pos_nodeid) { + fprintf(stderr, + "teac_Init: Expected Quadrics port id %d does not match real value %d!\n", + trans_host(buf), position.pos_nodeid); + teac_Close(result); + return NULL; + } +#elif (TEAC_DEVINFO_TYPE == 1) + if (trans_host(buf) != info.Position.NodeId) { + fprintf(stderr, + "teac_Init: Expected Quadrics port id %d does not match real value %d!\n", + trans_host(buf), info.Position.NodeId); + teac_Close(result); + return NULL; + } +#else + if (trans_host(buf) != info.NodeId) { + fprintf(stderr, + "teac_Init: Expected Quadrics port id %d does not match real value %d!\n", + trans_host(buf), info.NodeId); + teac_Close(result); + return NULL; + } +#endif + + if (elan3_attach(result->ctx, &(result->cap))) { + fprintf(stderr, "teac_Init: elan3_attach failed\n"); + teac_Close(result); + return(NULL); + } + + if (elan3_addvp(result->ctx, 0, &(result->cap))) { + fprintf(stderr, "teac_Init: elan3_addvp failed\n"); + teac_Close(result); + return(NULL); + } + + result->vp = elan3_process(result->ctx); + + if (! elan3_set_required_mappings (result->ctx)) { + fprintf(stderr, "teac_Init: elan3_set_required_mappings failed\n"); + teac_Close(result); + return(NULL); + } + + if (!(result->r_event= + (sdramaddr_t**)crAlloc( TEAC_CAP_ENTRIES(cap)*sizeof(sdramaddr_t*) ))) { + fprintf(stderr,"teac_Init: unable to allocate %d bytes!\n", + TEAC_CAP_ENTRIES(cap)*sizeof(sdramaddr_t*)); + teac_Close(result); + return(NULL); + } + if (!(result->r_event[0]= + (sdramaddr_t*)crAlloc( TEAC_CAP_ENTRIES(cap)*NUM_SEND_BUFFERS + * sizeof(sdramaddr_t) ))) { + fprintf(stderr,"teac_Init: unable to allocate %d bytes!\n", + TEAC_CAP_ENTRIES(cap)*NUM_SEND_BUFFERS*sizeof(sdramaddr_t)); + teac_Close(result); + return(NULL); + } + if (!(result->r_event[0][0]= + elan3_allocElan(result->ctx, E3_EVENT_ALIGN, + TEAC_CAP_ENTRIES(cap)*NUM_SEND_BUFFERS*sizeof(E3_BlockCopyEvent)))) { + perror("teac_Init: elan3_allocElan failed for r_event block"); + teac_Close(result); + return(NULL); + } + for (j=1; j<TEAC_CAP_ENTRIES(cap); j++) { + result->r_event[j]= result->r_event[0] + j*NUM_SEND_BUFFERS; + result->r_event[j][0]= + result->r_event[0][0]+j*NUM_SEND_BUFFERS*sizeof(E3_BlockCopyEvent); + } + for (j=0; j<TEAC_CAP_ENTRIES(cap); j++) + for (i=1; i<NUM_SEND_BUFFERS; i++) { + result->r_event[j][i]= + result->r_event[j][0] + i*sizeof(E3_BlockCopyEvent); + } +#if 0 + crDebug("r_event[0][0] is %x\n",(int)result->r_event[0][0]); +#endif + + if (!(result->m_rcv= + (volatile E3_uint32**)crAlloc( TEAC_CAP_ENTRIES(cap)*sizeof(E3_uint32*) ))) { + fprintf(stderr,"teac_Init: unable to allocate %d bytes!\n", + TEAC_CAP_ENTRIES(cap)*sizeof(E3_uint32*)); + teac_Close(result); + return(NULL); + } + if (!(result->m_rcv[0]= (volatile E3_uint32*) + elan3_allocMain(result->ctx, 0, + TEAC_CAP_ENTRIES(cap)*NUM_SEND_BUFFERS*sizeof(E3_uint32)))) { + perror("teac_Init: elan3_allocMain failed for m_rcv block"); + teac_Close(result); + return(NULL); + } + for (i=1; i<TEAC_CAP_ENTRIES(cap); i++) + result->m_rcv[i]= result->m_rcv[0] + i*NUM_SEND_BUFFERS; +#if 0 + crDebug("Base of m_rcv is %lx -> %lx\n", + (long)(result->m_rcv[0]), + (long)teac_main2elan(result->ctx,(void*)(result->m_rcv[0]))); +#endif + + if (!(result->mbuff= (teacMsg**)crAlloc( TEAC_CAP_ENTRIES(cap)*sizeof(teacMsg*) ))) { + fprintf(stderr,"teac_Init: unable to allocate %d bytes!\n", + TEAC_CAP_ENTRIES(cap)*sizeof(teacMsg*)); + teac_Close(result); + return(NULL); + } + if (!(result->mbuff[0]= (teacMsg*) + elan3_allocMain(result->ctx, 8, + TEAC_CAP_ENTRIES(cap)*NUM_SEND_BUFFERS*sizeof(teacMsg)))) { + perror("teac_Init: elan3_allocMain failed for mbuff block"); + teac_Close(result); + return(NULL); + } + for (i=1; i<TEAC_CAP_ENTRIES(cap); i++) + result->mbuff[i]= result->mbuff[0] + i*NUM_SEND_BUFFERS; +#if 0 + crDebug("Base of mbuff is %lx -> %lx\n", + (long)(result->mbuff[0]), + (long)teac_main2elan(result->ctx,result->mbuff[0])); +#endif + + if (!(result->dma= (E3_DMA_MAIN *) + elan3_allocMain(result->ctx, + E3_DMA_ALIGN, sizeof(E3_DMA_MAIN)))) { + perror("teac_Init: elan3_allocMain failed for dma"); + teac_Close(result); + return(NULL); + } + + if (!(result->e_dma= + elan3_allocElan(result->ctx, E3_DMA_ALIGN, sizeof(E3_DMA)))) { + perror("teac_Init: elan3_allocElan failed for e_dma"); + teac_Close(result); + return(NULL); + } + + if (!(result->s_event= + elan3_allocElan(result->ctx, E3_EVENT_ALIGN, + sizeof(E3_BlockCopyEvent)))) { + perror("teac_Init: elan3_allocElan failed for s_event"); + teac_Close(result); + return(NULL); + } +#if 0 + crDebug("s_event is %x\n",(int)result->s_event); +#endif + + if (!(result->sbuf_pull_event[0]= + elan3_allocElan(result->ctx, E3_EVENT_ALIGN, + NUM_SEND_BUFFERS*sizeof(E3_BlockCopyEvent)))) { + perror("teac_Init: elan3_allocElan failed for sbuf_pull_event block"); + teac_Close(result); + return(NULL); + } + for (i=1; i<NUM_SEND_BUFFERS; i++) + result->sbuf_pull_event[i]= + result->sbuf_pull_event[0]+i*sizeof(E3_BlockCopyEvent); + + if (!(result->rbuf_pull_event= + elan3_allocElan(result->ctx, E3_EVENT_ALIGN, + sizeof(E3_BlockCopyEvent)))) { + perror("teac_Init: elan3_allocElan failed for rbuf_pull_event"); + teac_Close(result); + return(NULL); + } + + if (!(result->m_snd= (E3_uint32 *) + elan3_allocMain(result->ctx, 0, + sizeof(E3_uint32)))) { + perror("teac_Init: elan3_allocMain failed for m_snd"); + teac_Close(result); + return(NULL); + } + + if (!(result->sbuf_ready[0]= (E3_uint32 *) + elan3_allocMain(result->ctx, 0, + NUM_SEND_BUFFERS*sizeof(E3_uint32)))) { + perror("teac_Init: elan3_allocMain failed for sbuf_ready block"); + teac_Close(result); + return(NULL); + } + for (i=1; i<NUM_SEND_BUFFERS; i++) + result->sbuf_ready[i]= result->sbuf_ready[0] + i; + + if (!(result->sendWrappers[0]= + (SBuffer*)crAlloc(NUM_SEND_BUFFERS*sizeof(SBuffer)))) { + fprintf(stderr,"teac_Init: unable to allocate %d bytes!\n", + NUM_SEND_BUFFERS*sizeof(SBuffer)); + teac_Close(result); + return(NULL); + } + for (i=1; i<NUM_SEND_BUFFERS; i++) + result->sendWrappers[i]= result->sendWrappers[0] + i; + + if (!(result->rbuf_ready= (E3_uint32 *) + elan3_allocMain(result->ctx, 0, + sizeof(E3_uint32)))) { + perror("teac_Init: elan3_allocMain failed for rbuf_ready"); + teac_Close(result); + return(NULL); + } + + for (i=0; i<NUM_SEND_BUFFERS; i++) { + char* buf; + if (!(buf= (char*)elan3_allocMain(result->ctx, 8, + E_BUFFER_INITIAL_SIZE))) { + perror("teac_Init: elan3_allocMain failed for buffer block"); + teac_Close(result); + return(NULL); + } + result->sendWrappers[i]->bufId= i; + result->sendWrappers[i]->totSize= E_BUFFER_INITIAL_SIZE; + result->sendWrappers[i]->validSize= 0; + result->sendWrappers[i]->buf= buf; + result->totalSendBufferBytesAllocated += result->sendWrappers[i]->totSize; + } + + elan3_initevent_word (result->ctx, + result->s_event, result->m_snd); + elan3_initevent_word (result->ctx, + result->rbuf_pull_event, result->rbuf_ready); + + for (i=0; i<NUM_SEND_BUFFERS; i++) { + elan3_initevent_word (result->ctx, + result->sbuf_pull_event[i], result->sbuf_ready[i]); + } + + for (j=0; j<TEAC_CAP_ENTRIES(cap); j++) + for (i=0; i<NUM_SEND_BUFFERS; i++) { + elan3_initevent_word (result->ctx, + result->r_event[j][i], &(result->m_rcv[j][i])); + } + + /* Get the message receive events ready to fire, in case something + * comes in before receive gets called. + */ + for (j=0; j<TEAC_CAP_ENTRIES(cap); j++) + for (i=0; i<NUM_SEND_BUFFERS; i++) { + elan3_primeevent(result->ctx, result->r_event[j][i], 1); + } + + /* Fire the sbuffer free events, so that the buffers look free when + * the first call to send happens. + */ + for (i=0; i<NUM_SEND_BUFFERS; i++) { + elan3_primeevent(result->ctx, result->sbuf_pull_event[i], 1); + elan3_setevent( result->ctx, result->sbuf_pull_event[i] ); + } + + /* And now we're ready to face the world. */ + elan3_block_inputter (result->ctx, 0); + + return(result); +} + +void teac_Close(Tcomm *tcomm) +{ + int i; + + if (tcomm) { + /* First we have to wait until all pending messages have been + * pulled (assuming they got initialized in the first place). + */ + if ((tcomm->sbuf_pull_event[0] != 0) && (tcomm->sbuf_ready[0] != NULL)) { + for (i=0; i<NUM_SEND_BUFFERS; i++) { + elan3_waitevent_word(tcomm->ctx, tcomm->sbuf_pull_event[i], + tcomm->sbuf_ready[i], 10); + } + crDebug("All TEAC messages have reported home!\n"); + } + elan3_block_inputter (tcomm->ctx, 1); + + if (tcomm->e_dma != 0) elan3_freeElan(tcomm->ctx, tcomm->e_dma); + if (tcomm->s_event != 0) elan3_freeElan(tcomm->ctx, tcomm->s_event); + if (tcomm->r_event != NULL) { + if (tcomm->r_event[0] != NULL) { + if (tcomm->r_event[0][0] != 0) elan3_freeElan(tcomm->ctx, + tcomm->r_event[0][0]); + crFree(tcomm->r_event[0]); + } + crFree(tcomm->r_event); + } + if (tcomm->sbuf_pull_event[0] != 0) + elan3_freeElan(tcomm->ctx, tcomm->sbuf_pull_event[0]); + if (tcomm->sendWrappers[0] != NULL) { + for (i=0; i<NUM_SEND_BUFFERS; i++) { + if (tcomm->sendWrappers[i]->buf != NULL) { + elan3_free(tcomm->ctx, tcomm->sendWrappers[i]->buf); + tcomm->totalSendBufferBytesAllocated -= + tcomm->sendWrappers[i]->totSize; + } + } + crFree(tcomm->sendWrappers[0]); + } + if (tcomm->rbuf_pull_event != 0) + elan3_freeElan(tcomm->ctx, tcomm->rbuf_pull_event); + if (tcomm->m_snd != NULL) elan3_free(tcomm->ctx, + (E3_uint32*)tcomm->m_snd); + if (tcomm->m_rcv != NULL) { + if (tcomm->m_rcv[0] != NULL) elan3_free(tcomm->ctx, + (E3_uint32*)tcomm->m_rcv[0]); + crFree(tcomm->m_rcv); + } + if (tcomm->sbuf_ready[0] != NULL) + elan3_free(tcomm->ctx, (E3_uint32*)tcomm->sbuf_ready[0]); + if (tcomm->rbuf_ready != NULL) elan3_free(tcomm->ctx, + (E3_uint32*)tcomm->rbuf_ready); + if (tcomm->mbuff != NULL) { + if (tcomm->mbuff[0] != NULL) elan3_free(tcomm->ctx, tcomm->mbuff[0]); + crFree(tcomm->mbuff); + } +#if (TEAC_CREATE_TYPE==2) + elan3_detach(tcomm->ctx); + elan3_fini(tcomm->ctx); +#else + /* elan3_detach and elan3_destroy seem to crash sometimes in + * these versions. + */ +#endif + } +} + +int teac_Select(Tcomm* tcomm, int *ids, int num_ids, int timeout) +{ + int id; + while (timeout-- > 0) { + if ((id= teac_Poll(tcomm, ids, num_ids)) >= 0) return id; + } + + return -1; +} + +int teac_Poll(Tcomm* tcomm, int *ids, int num_ids) +{ + int i; + int j; + + for (j=0; j<num_ids; j++) { + int index= (j+tcomm->poll_shift) % num_ids; + int thisId= ids[index]; + for (i=0; i<NUM_SEND_BUFFERS; i++) { + if (elan3_pollevent_word(tcomm->ctx, &(tcomm->m_rcv[thisId][i]), 1)) { +#ifdef never + tcomm->poll_shift= index; +#endif + return thisId; + } + } + } + + return -1; +} + +int teac_sendBufferAvailable(Tcomm* tcomm) +{ + int i; + for (i=0; i<NUM_SEND_BUFFERS; i++) { + if (elan3_pollevent_word(tcomm->ctx, tcomm->sbuf_ready[i], 1)) + return 1; + } + return 0; +} + +SBuffer* teac_getSendBuffer( Tcomm* tcomm, long size ) +{ + /* Find a free send buffer. We'll busy wait in this poll loop + * if necessary. + */ + int i= 0; + while (1) { + if (elan3_pollevent_word(tcomm->ctx, tcomm->sbuf_ready[i], + 1)) break; + if (++i == NUM_SEND_BUFFERS) i= 0; + } + /* We will use this buffer! */ +#if 0 + crDebug("Allocated message buffer %d\n",i); +#endif + *(tcomm->sbuf_ready[i])= 0; /* mark it busy */ + + /* If the associated chunk of memory is smaller than that requested, + * replace it with something larger. + */ + if (tcomm->sendWrappers[i]->totSize < size) { + tcomm->totalSendBufferBytesAllocated -= tcomm->sendWrappers[i]->totSize; + elan3_free( tcomm->ctx, tcomm->sendWrappers[i]->buf ); + if (!(tcomm->sendWrappers[i]->buf= + (char*)elan3_allocMain(tcomm->ctx, 8, size))) { + perror("teac_getSendBuffer: failed to grow send buffer"); + exit(-1); + } + tcomm->totalSendBufferBytesAllocated += size; + } + tcomm->sendWrappers[i]->totSize= size; + tcomm->sendWrappers[i]->validSize= 0; + return tcomm->sendWrappers[i]; +} + +SBuffer* teac_getUnreadySendBuffer( Tcomm* tcomm, long size ) +{ + SBuffer* result= NULL; + if (!(result= (SBuffer*)crAlloc(sizeof(SBuffer)))) { + fprintf(stderr,"libteac: read_node_map: unable to allocate %d bytes!\n", + sizeof(SBuffer)); + abort(); + } + result->bufId= -1; /* this marks it unready */ +#if 0 + crDebug("Allocated an unready message buffer!\n"); +#endif + + /* Allocate some DMA-able memory */ + if (!(result->buf= (char*)elan3_allocMain(tcomm->ctx, 8, size))) { + perror("teac_getUnreadySendBuffer: failed to allocate elan3 memory"); + exit(-1); + } + result->totSize= size; + result->validSize= 0; + tcomm->totalSendBufferBytesAllocated += result->totSize; + + return result; +} + +SBuffer* teac_makeSendBufferReady( Tcomm* tcomm, SBuffer* buf ) +{ + int i= 0; + /* If the input buffer is already ready, just return it */ + if (buf->bufId >= 0 && buf->bufId<NUM_SEND_BUFFERS) return buf; + + /* Find a free send buffer. We'll busy wait in this poll loop + * if necessary. + */ + while (1) { + if (elan3_pollevent_word(tcomm->ctx, tcomm->sbuf_ready[i], + 1)) break; + if (++i == NUM_SEND_BUFFERS) i= 0; + } + /* We will use this buffer! */ +#if 0 + crDebug("Allocated message buffer %d in makeSendBufferReady\n",i); +#endif + *(tcomm->sbuf_ready[i])= 0; /* mark it busy */ + + /* Substitute the unready payload for the old payload */ + tcomm->totalSendBufferBytesAllocated -= tcomm->sendWrappers[i]->totSize; + elan3_free( tcomm->ctx, tcomm->sendWrappers[i]->buf ); + tcomm->sendWrappers[i]->buf= buf->buf; + + tcomm->sendWrappers[i]->totSize= buf->totSize; + tcomm->sendWrappers[i]->validSize= buf->validSize; + crFree(buf); + return tcomm->sendWrappers[i]; + +} + +int teac_Send( Tcomm* tcomm, int* ids, int num_ids, SBuffer* buf, void *start ) +{ + int vp = tcomm->vp; + int iBuf; + int iDest; + teacMsg *msg; + + /* Complain loudly if this is an unready buffer */ + if (buf->bufId==-1) { + fprintf(stderr,"teac_Send: tried to send an unready buffer!\n"); + return 0; + } + + /* Reality check: is this one of my buffers? */ + if (buf->bufId<0 || buf->bufId>=NUM_SEND_BUFFERS) { + fprintf(stderr,"teac_Send: I don't know this buffer!\n"); + return 0; + } + + /* Reality check: did they write too much into the message? */ + if (buf->validSize > buf->totSize) { + fprintf(stderr,"teac_Send: message too large! (%ld vs %ld)\n", + buf->validSize, buf->totSize); + abort(); + return 0; + } + + iBuf= buf->bufId; + msg = &(tcomm->mbuff[vp][iBuf]); + msg->msgnum = tcomm->msgnum++; + msg->size = buf->validSize; + msg->host = vp; + if ( start != NULL ) + msg->mptr = teac_main2elan( tcomm->ctx, start ); + else + msg->mptr = teac_main2elan(tcomm->ctx, buf->buf); + msg->clr_event = elan3_sdram2elan(tcomm->ctx, tcomm->ctx->sdram, + tcomm->sbuf_pull_event[iBuf]); + msg->new= 1; + + /* Set up the parts of the DMA that are not specific to destination */ + tcomm->dma->dma_type = E3_DMA_TYPE(DMA_BYTE,DMA_WRITE,DMA_NORMAL,0); + tcomm->dma->dma_size = sizeof(teacMsg); + tcomm->dma->dma_srcEvent = + elan3_sdram2elan(tcomm->ctx,tcomm->ctx->sdram,tcomm->s_event); + tcomm->dma->dma_source = teac_main2elan(tcomm->ctx, msg); + elan3_primeevent(tcomm->ctx, tcomm->sbuf_pull_event[buf->bufId], num_ids); + + /* Send this message off to each destination */ + for (iDest=0; iDest<num_ids; iDest++) { + tcomm->dma->dma_srcCookieVProc = + elan3_local_cookie(tcomm->ctx, vp, ids[iDest]); + tcomm->dma->dma_destCookieVProc = ids[iDest]; + tcomm->dma->dma_destEvent = + elan3_sdram2elan(tcomm->ctx,tcomm->ctx->sdram, + tcomm->r_event[vp][iBuf]) + + sdramAddrBase[(ids[iDest]/NUM_SEND_BUFFERS) + tcomm->lhost] + - sdramAddrBase[(vp/NUM_SEND_BUFFERS) + tcomm->lhost]; + tcomm->dma->dma_dest = teac_main2elan(tcomm->ctx, msg) + + elanAddrBase[(ids[iDest]/NUM_SEND_BUFFERS) + tcomm->lhost] + - elanAddrBase[(vp/NUM_SEND_BUFFERS) + tcomm->lhost]; + elan3_primeevent(tcomm->ctx, tcomm->s_event, 1); + *(tcomm->m_snd)= 0; + elan3_putdma_main(tcomm->ctx, tcomm->dma, tcomm->e_dma); +#if 0 + crDebug("DMA dest event %x, dest mem %lx\n", + tcomm->dma->dma_destEvent, + (long)tcomm->dma->dma_dest); + crDebug("Mem shifts are %x %x based on %d %d\n", + elanAddrBase[(ids[iDest]/NUM_SEND_BUFFERS) + tcomm->lhost], + elanAddrBase[(vp/NUM_SEND_BUFFERS) + tcomm->lhost], + ids[iDest],vp); + crDebug("Send msg %d in buffer %d to %d (list index %d)...", + msg->msgnum,iBuf, ids[iDest],iDest); +#endif + elan3_waitevent_word(tcomm->ctx, + tcomm->s_event, tcomm->m_snd, ELAN_WAIT_EVENT); +#if 0 + crDebug("message away!\n"); +#endif + } + return 1; +} + +RBuffer* teac_Recv(Tcomm* tcomm, int id) +{ + int vp = tcomm->vp; + RBuffer* result= NULL; + int iEvent= 0; + int iBuf= 0; + int i; + int lowestMsgnum; + + /* poll until we get an incoming message from the given sender */ + while (1) { + if (elan3_pollevent_word(tcomm->ctx, &(tcomm->m_rcv[id][iEvent]), 1)) + break; + if (++iEvent == NUM_SEND_BUFFERS) iEvent= 0; + } + + /* This may not be the earliest event, however. */ + lowestMsgnum= -1; + for (i=0; i<NUM_SEND_BUFFERS; i++) { +#if 0 + crDebug("Testing for new msg at %lx -> %lx\n", + (long)&(tcomm->mbuff[id][i]), + (long)teac_main2elan(tcomm->ctx,(void*)(&tcomm->mbuff[id][i]))); +#endif + if (tcomm->mbuff[id][i].new) { + if ((lowestMsgnum < 0) + || (tcomm->mbuff[id][i].msgnum < (E3_uint32)lowestMsgnum)) { + lowestMsgnum= tcomm->mbuff[id][i].msgnum; + iBuf= i; + } + } + } + if (lowestMsgnum<0) { + fprintf(stderr, + "teac_Recv: internal error: can't find message I just got!\n"); + return NULL; + } + tcomm->mbuff[id][iBuf].new= 0; + tcomm->m_rcv[id][iBuf]= 0; + elan3_primeevent(tcomm->ctx, tcomm->r_event[id][iBuf],1); +#if 0 + crDebug("got msg %d in buffer %d from %d!\n", + tcomm->mbuff[id][iBuf].msgnum, iBuf, id); +#endif + + /* Make some space to put the message when it arrives */ + if (!(result= (RBuffer*)crAlloc(sizeof(RBuffer)))) { + fprintf(stderr,"teac_Recv: unable to allocate %d bytes!\n", + sizeof(RBuffer)); + return NULL; + } +#if 0 + if (!(result->buf= (void*)elan3_allocMain(tcomm->ctx, 8,E_BUFFER_SIZE))) { + perror("teac_Recv: elan3_allocMain failed for buffer"); + return(NULL); + } + result->totSize= E_BUFFER_SIZE; +#endif + if (!(result->buf= (void*)elan3_allocMain(tcomm->ctx, 8, + tcomm->mbuff[id][iBuf].size))) { + perror("teac_Recv: elan3_allocMain failed for buffer"); + return(NULL); + } + result->totSize= tcomm->mbuff[id][iBuf].size; + tcomm->totalRecvBufferBytesAllocated+= result->totSize; + result->validSize= tcomm->mbuff[id][iBuf].size; + result->from= tcomm->mbuff[id][iBuf].host; + result->senderMsgnum= tcomm->mbuff[id][iBuf].msgnum; + + /* Set up the DMA to pull the message */ + tcomm->dma->dma_type = E3_DMA_TYPE(DMA_BYTE,DMA_READ,DMA_NORMAL,0); + tcomm->dma->dma_size = tcomm->mbuff[id][iBuf].size; + tcomm->dma->dma_source = tcomm->mbuff[id][iBuf].mptr; + tcomm->dma->dma_dest = teac_main2elan(tcomm->ctx,result->buf); + tcomm->dma->dma_srcCookieVProc = + elan3_remote_cookie(tcomm->ctx, vp, tcomm->mbuff[id][iBuf].host); + tcomm->dma->dma_destCookieVProc = + elan3_local_cookie(tcomm->ctx, vp, tcomm->mbuff[id][iBuf].host); + tcomm->dma->dma_srcEvent = tcomm->mbuff[id][iBuf].clr_event; + tcomm->dma->dma_destEvent = + elan3_sdram2elan(tcomm->ctx,tcomm->ctx->sdram, + tcomm->rbuf_pull_event); + + + elan3_primeevent(tcomm->ctx, tcomm->rbuf_pull_event,1); + *(tcomm->rbuf_ready)= 0; + elan3_getdma_main(tcomm->ctx, tcomm->dma, tcomm->e_dma); + elan3_waitevent_word(tcomm->ctx, + tcomm->rbuf_pull_event, + tcomm->rbuf_ready, ELAN_WAIT_EVENT); + + return(result); +} + +int teac_Dispose( Tcomm* tcomm, RBuffer* buf ) +{ + tcomm->totalRecvBufferBytesAllocated -= buf->totSize; + elan3_free(tcomm->ctx, buf->buf); + crFree(buf); + return 0; +} + +char* teac_getTcommString(Tcomm *c, char* buf, int buflen) +{ + snprintf(buf,buflen-1, + "<vp= %d, host range %d-%d, ctxnum range %d-%d, msg %d>", + c->vp,c->lhost,c->hhost,c->lctx,c->hctx,c->msgnum); + buf[buflen-1]= '\0'; + return buf; +} + +char* teac_getConnString(Tcomm *c, int id, char* buf, int buflen) +{ + int rel_rank= id%4; + int node= ((id-rel_rank)/4) + c->lhost; + snprintf(buf,buflen-1,"vp %d, <%s>:%d",id, + hosts[hostsIndex[node]].name,rel_rank); + buf[buflen-1]= '\0'; +#if 0 + crDebug("getConnString: lookup id %d -> %d %d -> %s\n",id,rel_rank,node,buf); +#endif + return buf; +} + +int teac_getConnId(Tcomm *c, const char* host, int rank) +{ + int node= trans_host(host); +#if 0 + crDebug("getConnId: <%s> %d %d maps to %d %d\n", + host, rank, c->lhost, node, (4*(node - c->lhost) + (rank%4))); +#endif + return (4*(node - c->lhost) + (rank%4)); +} + +int teac_getHostInfo(Tcomm *c, char* host, const int hostLength, + int* railMask, int *nodeId, + long* sdramBaseAddr, long* elanBaseAddr) +{ +#if (TEAC_DEVINFO_TYPE == 2) + ELAN_DEVINFO info; + ELAN_POSITION position; +#else + ELAN3_DEVINFO info; +#endif + char* here; + + if (gethostname(host,hostLength-1)) { + perror("Can't get my own host name"); + return 0; + } + host[hostLength-1]= '\0'; + if ((here= crStrchr(host,'.')) != NULL) *here= '\0'; + +#if (TEAC_DEVINFO_TYPE == 2) + (void)elan3_get_devinfo(c->ctx, &info); + (void)elan3_get_position(c->ctx, &position); + *nodeId= position.pos_nodeid; +#elif (TEAC_DEVINFO_TYPE == 1) + elan3_devinfo(0, &info); + *nodeId= info.Position.NodeId; +#else + elan3_devinfo(0, &info); + *nodeId= info.NodeId; +#endif + +#if (TEAC_CAP_TYPE == 2) + *railMask= c->cap.cap_railmask; +#else + *railMask= c->cap.RailMask; +#endif + + *sdramBaseAddr= (int)c->r_event[0][0]; + + *elanBaseAddr= (long)teac_main2elan(c->ctx,(void*)c->m_rcv[0]); + return 1; +} |