From 373d663fd06006052e34b4c7e5fd7281567ac3b2 Mon Sep 17 00:00:00 2001 From: Stefan Lankes Date: Tue, 19 Apr 2011 07:17:07 -0700 Subject: [PATCH] add iRCCE code --- arch/x86/include/asm/iRCCE.h | 154 ++++++++++++ arch/x86/include/asm/iRCCE_lib.h | 39 +++ arch/x86/scc/Makefile | 2 +- arch/x86/scc/iRCCE_admin.c | 67 ++++++ arch/x86/scc/iRCCE_get.c | 85 +++++++ arch/x86/scc/iRCCE_irecv.c | 393 +++++++++++++++++++++++++++++++ arch/x86/scc/iRCCE_isend.c | 355 ++++++++++++++++++++++++++++ arch/x86/scc/iRCCE_put.c | 87 +++++++ arch/x86/scc/iRCCE_recv.c | 190 +++++++++++++++ arch/x86/scc/iRCCE_send.c | 165 +++++++++++++ arch/x86/scc/iRCCE_synch.c | 127 ++++++++++ 11 files changed, 1663 insertions(+), 1 deletion(-) create mode 100644 arch/x86/include/asm/iRCCE.h create mode 100644 arch/x86/include/asm/iRCCE_lib.h create mode 100644 arch/x86/scc/iRCCE_admin.c create mode 100644 arch/x86/scc/iRCCE_get.c create mode 100644 arch/x86/scc/iRCCE_irecv.c create mode 100644 arch/x86/scc/iRCCE_isend.c create mode 100644 arch/x86/scc/iRCCE_put.c create mode 100644 arch/x86/scc/iRCCE_recv.c create mode 100644 arch/x86/scc/iRCCE_send.c create mode 100644 arch/x86/scc/iRCCE_synch.c diff --git a/arch/x86/include/asm/iRCCE.h b/arch/x86/include/asm/iRCCE.h new file mode 100644 index 00000000..8b878bfd --- /dev/null +++ b/arch/x86/include/asm/iRCCE.h @@ -0,0 +1,154 @@ +// +// Copyright 2010 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// [2010-10-25] added support for non-blocking send/recv operations +// - iRCCE_isend(), ..._test(), ..._wait(), ..._push() +// - iRCCE_irecv(), ..._test(), ..._wait(), ..._push() +// by Carsten Clauss, Chair for Operating Systems, +// RWTH Aachen University +// +// [2010-11-12] extracted non-blocking code into separate library +// by Carsten Scholtes, University of Bayreuth +// +// [2010-12-09] added functions for a convenient handling of multiple +// pending non-blocking requests +// by Jacek Galowicz, Chair for Operating Systems +// RWTH Aachen University +// +#ifndef IRCCE_H +#define IRCCE_H + +#include + +#define iRCCE_SUCCESS RCCE_SUCCESS +#define iRCCE_PENDING -1 +#define iRCCE_RESERVED -2 +#define iRCCE_NOT_ENQUEUED -3 + +typedef struct _iRCCE_SEND_REQUEST { + char *privbuf; // source buffer in local private memory (send buffer) + t_vcharp combuf; // intermediate buffer in MPB + size_t chunk; // size of MPB available for this message (bytes) + RCCE_FLAG *ready; // flag indicating whether receiver is ready + RCCE_FLAG *sent; // flag indicating whether message has been sent by source + size_t size; // size of message (bytes) + int dest; // UE that will receive the message + + size_t wsize; // offset within send buffer when putting in "chunk" bytes + size_t remainder; // bytes remaining to be sent + size_t nbytes; // number of bytes to be sent in single RCCE_put call + char *bufptr; // running pointer inside privbuf for current location + + int label; // jump/goto label for the reentrance of the respective poll function + int finished; // flag that indicates whether the request has already been finished + + struct _iRCCE_SEND_REQUEST *next; +} iRCCE_SEND_REQUEST; + + +typedef struct _iRCCE_RECV_REQUEST { + char *privbuf; // source buffer in local private memory (send buffer) + t_vcharp combuf; // intermediate buffer in MPB + size_t chunk; // size of MPB available for this message (bytes) + RCCE_FLAG *ready; // flag indicating whether receiver is ready + RCCE_FLAG *sent; // flag indicating whether message has been sent by source + size_t size; // size of message (bytes) + int source; // UE that will send the message + + size_t wsize; // offset within send buffer when putting in "chunk" bytes + size_t remainder; // bytes remaining to be sent + size_t nbytes; // number of bytes to be sent in single RCCE_put call + char *bufptr; // running pointer inside privbuf for current location + + int label; // jump/goto label for the reentrance of the respective poll function + int finished; // flag that indicates whether the request has already been finished + int started; // flag that indicates whether message parts have already been received + + struct _iRCCE_RECV_REQUEST *next; +} iRCCE_RECV_REQUEST; + +#define iRCCE_WAIT_LIST_RECV_TYPE 0 +#define iRCCE_WAIT_LIST_SEND_TYPE 1 + +typedef struct _iRCCE_WAIT_LISTELEM { + int type; + struct _iRCCE_WAIT_LISTELEM * next; + void * req; +} iRCCE_WAIT_LISTELEM; + +typedef struct _iRCCE_WAIT_LIST { + iRCCE_WAIT_LISTELEM * first; + iRCCE_WAIT_LISTELEM * last; +} iRCCE_WAIT_LIST; + + +/////////////////////////////////////////////////////////////// +// +// THE iRCCE API: +// +// Initialize function: +int iRCCE_init(void); +// +// Non-blocking send/recv functions: +int iRCCE_isend(char *, size_t, int, iRCCE_SEND_REQUEST *); +int iRCCE_isend_test(iRCCE_SEND_REQUEST *, int *); +int iRCCE_isend_wait(iRCCE_SEND_REQUEST *); +int iRCCE_isend_push(void); +int iRCCE_irecv(char *, size_t, int, iRCCE_RECV_REQUEST *); +int iRCCE_irecv_test(iRCCE_RECV_REQUEST *, int *); +int iRCCE_irecv_wait(iRCCE_RECV_REQUEST *); +int iRCCE_irecv_push(void); +// +// Blocking but pipelined send/recv functions: +int iRCCE_send(char *, size_t, int); +int iRCCE_recv(char *, size_t, int); +// +// SCC-customized put/get and memcpy functions: +int iRCCE_put(t_vcharp, t_vcharp, int, int); +int iRCCE_get(t_vcharp, t_vcharp, int, int); +void* iRCCE_memcpy_put(void*, const void*, size_t); +void* iRCCE_memcpy_get(void*, const void*, size_t); +// +// Wait/test-all/any functions: +void iRCCE_init_wait_list(iRCCE_WAIT_LIST*); +void iRCCE_add_to_wait_list(iRCCE_WAIT_LIST*, iRCCE_SEND_REQUEST *, iRCCE_RECV_REQUEST *); +int iRCCE_test_all(iRCCE_WAIT_LIST*, int *); +int iRCCE_wait_all(iRCCE_WAIT_LIST*); +int iRCCE_test_any(iRCCE_WAIT_LIST*, iRCCE_SEND_REQUEST **, iRCCE_RECV_REQUEST **); +int iRCCE_wait_any(iRCCE_WAIT_LIST*, iRCCE_SEND_REQUEST **, iRCCE_RECV_REQUEST **); +// +// Cancel functions for yet not started non-blocking requests: +int iRCCE_isend_cancel(iRCCE_SEND_REQUEST *, int *); +int iRCCE_irecv_cancel(iRCCE_RECV_REQUEST *, int *); +// +/////////////////////////////////////////////////////////////// +// +// Just for for convenience: +#if 1 +#define RCCE_isend iRCCE_isend +#define RCCE_isend_test iRCCE_isend_test +#define RCCE_isend_wait iRCCE_isend_wait +#define RCCE_isend_push iRCCE_isend_push +#define RCCE_irecv iRCCE_irecv +#define RCCE_irecv_test iRCCE_irecv_test +#define RCCE_irecv_wait iRCCE_irecv_wait +#define RCCE_irecv_push iRCCE_irecv_push +#define RCCE_SEND_REQUEST iRCCE_SEND_REQUEST +#define RCCE_RECV_REQUEST iRCCE_RECV_REQUEST +#endif +/////////////////////////////////////////////////////////////// + +#endif + diff --git a/arch/x86/include/asm/iRCCE_lib.h b/arch/x86/include/asm/iRCCE_lib.h new file mode 100644 index 00000000..0d8b4e16 --- /dev/null +++ b/arch/x86/include/asm/iRCCE_lib.h @@ -0,0 +1,39 @@ +// +// Copyright 2010 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// [2010-10-25] added support for non-blocking send/recv operations +// - iRCCE_isend(), ..._test(), ..._wait(), ..._push() +// - iRCCE_irecv(), ..._test(), ..._wait(), ..._push() +// by Carsten Clauss, Chair for Operating Systems, +// RWTH Aachen University +// +// [2010-11-12] extracted non-blocking code into separate library +// by Carsten Scholtes +// +#ifndef IRCCE_LIB_H +#define IRCCE_LIB_H +#include +#include + +extern iRCCE_SEND_REQUEST* iRCCE_isend_queue; +extern iRCCE_RECV_REQUEST* iRCCE_irecv_queue[RCCE_MAXNP]; +#ifdef _OPENMP +#pragma omp threadprivate (iRCCE_isend_queue, iRCCE_irecv_queue) +#endif + +int iRCCE_test_flag(RCCE_FLAG, RCCE_FLAG_STATUS, int *); + +#endif + diff --git a/arch/x86/scc/Makefile b/arch/x86/scc/Makefile index d4866f2b..f32bc11f 100644 --- a/arch/x86/scc/Makefile +++ b/arch/x86/scc/Makefile @@ -1,4 +1,4 @@ -C_source := scc_init.c SCC_API.c RCCE_malloc.c RCCE_shmalloc.c RCCE_debug.c RCCE_qsort.c RCCE_DCMflush.c RCCE_send.c RCCE_recv.c RCCE_flags.c RCCE_comm.c RCCE_put.c RCCE_get.c RCCE_synch.c RCCE_bcast.c RCCE_admin.c # RCCE_power_management.c +C_source := scc_init.c SCC_API.c iRCCE_admin.c iRCCE_send.c iRCCE_isend.c iRCCE_irecv.c iRCCE_recv.c iRCCE_get.c iRCCE_put.c iRCCE_synch.c RCCE_malloc.c RCCE_shmalloc.c RCCE_debug.c RCCE_qsort.c RCCE_DCMflush.c RCCE_send.c RCCE_recv.c RCCE_flags.c RCCE_comm.c RCCE_put.c RCCE_get.c RCCE_synch.c RCCE_bcast.c RCCE_admin.c # RCCE_power_management.c ASM_source := MODULE := arch_x86_scc diff --git a/arch/x86/scc/iRCCE_admin.c b/arch/x86/scc/iRCCE_admin.c new file mode 100644 index 00000000..c61d66b9 --- /dev/null +++ b/arch/x86/scc/iRCCE_admin.c @@ -0,0 +1,67 @@ +//*************************************************************************************** +// Administrative routines. +//*************************************************************************************** +// +// Author: Rob F. Van der Wijngaart +// Intel Corporation +// Date: 008/30/2010 +// +//*************************************************************************************** +// +// +// Copyright 2010 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// [2010-10-25] added support for non-blocking send/recv operations +// - iRCCE_isend(), ..._test(), ..._wait(), ..._push() +// - iRCCE_irecv(), ..._test(), ..._wait(), ..._push() +// by Carsten Clauss, Chair for Operating Systems, +// RWTH Aachen University +// +// [2010-11-12] extracted non-blocking code into separate library +// by Carsten Scholtes +// +// [2011-02-21] added support for multiple incoming queues +// (one recv queue per remote rank) +// + +#include + +#ifdef CONFIG_ROCKCREEK + +#include + +// send request queue +iRCCE_SEND_REQUEST* iRCCE_isend_queue; +// recv request queue +iRCCE_RECV_REQUEST* iRCCE_irecv_queue[RCCE_MAXNP]; + +//-------------------------------------------------------------------------------------- +// FUNCTION: iRCCE_init +//-------------------------------------------------------------------------------------- +// initialize the library +//-------------------------------------------------------------------------------------- +int iRCCE_init(void) { + int i; + + for(i=0; i +#include + +#ifdef CONFIG_ROCKCREEK + +#include + +#ifdef COPPERRIDGE +#include "scc_memcpy.h" +#endif + +void* iRCCE_memcpy_get(void *dest, const void *src, size_t count) +{ +#ifdef COPPERRIDGE + return memcpy_from_mpb(dest, src, count); +#else + return memcpy(dest, src, count); +#endif +} + +//-------------------------------------------------------------------------------------- +// FUNCTION: iRCCE_get +//-------------------------------------------------------------------------------------- +// copy data from address "source" in the remote MPB to address "target" in either the +// local MPB, or in the calling UE's private memory. We do not test to see if a move +// into the calling UE's private memory stays within allocated memory * +//-------------------------------------------------------------------------------------- +int iRCCE_get( + t_vcharp target, // target buffer, MPB or private memory + t_vcharp source, // source buffer, MPB + int num_bytes, // number of bytes to copy (must be multiple of cache line size + int ID // rank of source UE + ) { + + // in non-GORY mode we only need to retain the MPB source shift; we + // already know the source is in the MPB, not private memory + source = RCCE_comm_buffer[ID]+(source-RCCE_comm_buffer[RCCE_IAM]); + + // do the actual copy, making sure we copy fresh data +#ifdef _OPENMP + #pragma omp flush +#endif + RC_cache_invalidate(); + + iRCCE_memcpy_get((void *)target, (void *)source, num_bytes); + + // flush data to make sure it is visible to all threads; cannot use a flush list + // because it concerns malloced space +#ifdef _OPENMP + #pragma omp flush +#endif + return(iRCCE_SUCCESS); +} + +#endif diff --git a/arch/x86/scc/iRCCE_irecv.c b/arch/x86/scc/iRCCE_irecv.c new file mode 100644 index 00000000..e7d5ad1d --- /dev/null +++ b/arch/x86/scc/iRCCE_irecv.c @@ -0,0 +1,393 @@ +//*************************************************************************************** +// Synchronized receive routines. +//*************************************************************************************** +// +// Author: Rob F. Van der Wijngaart +// Intel Corporation +// Date: 008/30/2010 +// +//*************************************************************************************** +// +// Copyright 2010 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// [2010-10-25] added support for non-blocking send/recv operations +// - iRCCE_isend(), ..._test(), ..._wait(), ..._push() +// - iRCCE_irecv(), ..._test(), ..._wait(), ..._push() +// by Carsten Clauss, Chair for Operating Systems, +// RWTH Aachen University +// +// [2010-11-12] extracted non-blocking code into separate library +// by Carsten Scholtes +// +// [2010-12-09] added cancel functions for non-blocking send/recv requests +// by Carsten Clauss +// +// [2011-02-21] added support for multiple incoming queues +// (one recv queue per remote rank) +// + +#include +#include + +#ifdef CONFIG_ROCKCREEK + +#include + +static int iRCCE_push_recv_request(iRCCE_RECV_REQUEST *request) { + + char padline[RCCE_LINE_SIZE]; // copy buffer, used if message not multiple of line size + int test; // flag for calling iRCCE_test_flag() + + if(request->finished) return(iRCCE_SUCCESS); + + if(request->label == 1) goto label1; + if(request->label == 2) goto label2; + if(request->label == 3) goto label3; + + // receive data in units of available chunk size of MPB + for (; request->wsize < (request->size / request->chunk) * request->chunk; request->wsize += request->chunk) { + request->bufptr = request->privbuf + request->wsize; + request->nbytes = request->chunk; +label1: + iRCCE_test_flag(*(request->sent), RCCE_FLAG_SET, &test); + if(!test) { + request->label = 1; + return(iRCCE_PENDING); + } + request->started = 1; + + RCCE_flag_write(request->sent, RCCE_FLAG_UNSET, RCCE_IAM); + // copy data from source's MPB space to private memory + iRCCE_get((t_vcharp)request->bufptr, request->combuf, request->nbytes, request->source); + + // tell the source I have moved data out of its comm buffer + RCCE_flag_write(request->ready, RCCE_FLAG_SET, request->source); + } + + request->remainder = request->size % request->chunk; + // if nothing is left over, we are done + if (!request->remainder) { + request->finished = 1; + return(iRCCE_SUCCESS); + } + + // receive remainder of data--whole cache lines + request->bufptr = request->privbuf + (request->size / request->chunk) * request->chunk; + request->nbytes = request->remainder - request->remainder % RCCE_LINE_SIZE; + if (request->nbytes) { +label2: + iRCCE_test_flag(*(request->sent), RCCE_FLAG_SET, &test); + if(!test) { + request->label = 2; + return(iRCCE_PENDING); + } + request->started = 1; + + RCCE_flag_write(request->sent, RCCE_FLAG_UNSET, RCCE_IAM); + // copy data from source's MPB space to private memory + iRCCE_get((t_vcharp)request->bufptr, request->combuf, request->nbytes, request->source); + + // tell the source I have moved data out of its comm buffer + RCCE_flag_write(request->ready, RCCE_FLAG_SET, request->source); + } + + request->remainder = request->size % request->chunk; + request->remainder = request->remainder % RCCE_LINE_SIZE; + if (!request->remainder) { + request->finished = 1; + return(iRCCE_SUCCESS); + } + + // remainder is less than cache line. This must be copied into appropriately sized + // intermediate space before exact number of bytes get copied to the final destination + request->bufptr = request->privbuf + (request->size / request->chunk) * request->chunk + request->nbytes; + request->nbytes = RCCE_LINE_SIZE; +label3: + iRCCE_test_flag(*(request->sent), RCCE_FLAG_SET, &test); + if(!test) { + request->label = 3; + return(iRCCE_PENDING); + } + request->started = 1; + + RCCE_flag_write(request->sent, RCCE_FLAG_UNSET, RCCE_IAM); + // copy data from source's MPB space to private memory + iRCCE_get((t_vcharp)padline, request->combuf, request->nbytes, request->source); + memcpy(request->bufptr,padline,request->remainder); + + // tell the source I have moved data out of its comm buffer + RCCE_flag_write(request->ready, RCCE_FLAG_SET, request->source); + + request->finished = 1; + return(iRCCE_SUCCESS); +} + +static void iRCCE_init_recv_request( + char *privbuf, // source buffer in local private memory (send buffer) + t_vcharp combuf, // intermediate buffer in MPB + size_t chunk, // size of MPB available for this message (bytes) + RCCE_FLAG *ready, // flag indicating whether receiver is ready + RCCE_FLAG *sent, // flag indicating whether message has been sent by source + size_t size, // size of message (bytes) + int source, // UE that will send the message + iRCCE_RECV_REQUEST *request + ) { + + request->privbuf = privbuf; + request->combuf = combuf; + request->chunk = chunk; + request->ready = ready; + request->sent = sent; + request->size = size; + request->source = source; + + request->wsize = 0; + request->remainder = 0; + request->nbytes = 0; + request->bufptr = NULL; + + request->label = 0; + request->finished = 0; + request->started = 0; + + request->next = NULL; + + return; +} + + +//-------------------------------------------------------------------------------------- +// FUNCTION: iRCCE_irecv +//-------------------------------------------------------------------------------------- +// non-blocking recv function; returns an handle of type iRCCE_RECV_REQUEST +//-------------------------------------------------------------------------------------- +static iRCCE_RECV_REQUEST blocking_irecv_request; +int iRCCE_irecv(char *privbuf, size_t size, int source, iRCCE_RECV_REQUEST *request) { + + if(request == NULL) request = &blocking_irecv_request; + + if (source<0 || source >= RCCE_NP) + return(RCCE_error_return(RCCE_debug_comm,RCCE_ERROR_ID)); + else { + iRCCE_init_recv_request(privbuf, RCCE_buff_ptr, RCCE_chunk, + &RCCE_ready_flag[RCCE_IAM], &RCCE_sent_flag[source], + size, source, request); + + if(iRCCE_irecv_queue[source] == NULL) { + + if(iRCCE_push_recv_request(request) == iRCCE_SUCCESS) { + return(iRCCE_SUCCESS); + } + else { + iRCCE_irecv_queue[source] = request; + + if(request == &blocking_irecv_request) { + iRCCE_irecv_wait(request); + return(iRCCE_SUCCESS); + } + + return(iRCCE_PENDING); + } + } + else { + if(iRCCE_irecv_queue[source]->next == NULL) { + iRCCE_irecv_queue[source]->next = request; + } + else { + iRCCE_RECV_REQUEST *run = iRCCE_irecv_queue[source]; + while(run->next != NULL) run = run->next; + run->next = request; + } + + if(request == &blocking_irecv_request) { + iRCCE_irecv_wait(request); + return(iRCCE_SUCCESS); + } + + return(iRCCE_RESERVED); + } + } +} + +//-------------------------------------------------------------------------------------- +// FUNCTION: iRCCE_irecv_test +//-------------------------------------------------------------------------------------- +// test function for completion of the requestes non-blocking recv operation +// Just provide NULL instead of the testvar if you don't need it +//-------------------------------------------------------------------------------------- +int iRCCE_irecv_test(iRCCE_RECV_REQUEST *request, int *test) { + + int source; + + if(request == NULL) { + + if(iRCCE_irecv_push() == iRCCE_SUCCESS) { + if (test) (*test) = 1; + return(iRCCE_SUCCESS); + } + else { + if (test) (*test) = 0; + return(iRCCE_PENDING); + } + } + + source = request->source; + + if(request->finished) { + if (test) (*test) = 1; + return(iRCCE_SUCCESS); + } + + if(iRCCE_irecv_queue[source] != request) { + if (test) (*test) = 0; + return(iRCCE_RESERVED); + } + + iRCCE_push_recv_request(request); + + if(request->finished) { + iRCCE_irecv_queue[source] = request->next; + + if (test) (*test) = 1; + return(iRCCE_SUCCESS); + } + + if (test) (*test) = 0; + return(iRCCE_PENDING); +} + + +//-------------------------------------------------------------------------------------- +// FUNCTION: iRCCE_irecv_push +//-------------------------------------------------------------------------------------- +// progress function for pending requests in the irecv queue +//-------------------------------------------------------------------------------------- +static int iRCCE_irecv_push_source(int source) { + + iRCCE_RECV_REQUEST *request = iRCCE_irecv_queue[source]; + + if(request == NULL) { + return(iRCCE_SUCCESS); + } + + if(request->finished) { + return(iRCCE_SUCCESS); + } + + iRCCE_push_recv_request(request); + + if(request->finished) { + iRCCE_irecv_queue[source] = request->next; + return(iRCCE_SUCCESS); + } + + return(iRCCE_PENDING); +} + +int iRCCE_irecv_push(void) { + + int i, j; + int retval = iRCCE_SUCCESS; + + for(i=0; ifinished) { + iRCCE_irecv_push(); + iRCCE_isend_push(); + } + } + else { + do { + iRCCE_isend_push(); + } + while( iRCCE_irecv_push() != iRCCE_SUCCESS ); + } + + return(iRCCE_SUCCESS); +} + + +//-------------------------------------------------------------------------------------- +// FUNCTION: iRCCE_irecv_cancel +//-------------------------------------------------------------------------------------- +// try to cancel a pending non-blocking recv request +//-------------------------------------------------------------------------------------- +int iRCCE_irecv_cancel(iRCCE_RECV_REQUEST *request, int *test) { + + int source; + iRCCE_RECV_REQUEST *run; + + if( (request == NULL) || (request->finished) ) { + if (test) (*test) = 0; + return iRCCE_NOT_ENQUEUED; + } + + source = request->source; + + if(iRCCE_irecv_queue[source] == NULL) { + if (test) (*test) = 0; + return iRCCE_NOT_ENQUEUED; + } + + if(iRCCE_irecv_queue[source] == request) { + + // have parts of the message already been received? + if(request->started) { + if (test) (*test) = 0; + return iRCCE_PENDING; + } + else { + // no, thus request can be canceld just in time: + iRCCE_irecv_queue[source] = request->next; + if (test) (*test) = 1; + return iRCCE_SUCCESS; + } + } + + for(run = iRCCE_irecv_queue[source]; run->next != NULL; run = run->next) { + + // request found --> remove it from recv queue: + if(run->next == request) { + + run->next = run->next->next; + + if (test) (*test) = 1; + return iRCCE_SUCCESS; + } + } + + if (test) (*test) = 0; + return iRCCE_NOT_ENQUEUED; +} + +#endif diff --git a/arch/x86/scc/iRCCE_isend.c b/arch/x86/scc/iRCCE_isend.c new file mode 100644 index 00000000..18c9dca0 --- /dev/null +++ b/arch/x86/scc/iRCCE_isend.c @@ -0,0 +1,355 @@ +//*************************************************************************************** +// Non-blocking send routines. +//*************************************************************************************** +// +// Author: Rob F. Van der Wijngaart +// Intel Corporation +// Date: 008/30/2010 +// +//*************************************************************************************** +// +// Copyright 2010 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// [2010-10-25] added support for non-blocking send/recv operations +// - iRCCE_isend(), ..._test(), ..._wait(), ..._push() +// - iRCCE_irecv(), ..._test(), ..._wait(), ..._push() +// by Carsten Clauss, Chair for Operating Systems, +// RWTH Aachen University +// +// [2010-11-12] extracted non-blocking code into separate library +// by Carsten Scholtes +// +// [2010-12-09] added cancel functions for non-blocking send/recv requests +// by Carsten Clauss +// + +#include +#include + +#ifdef CONFIG_ROCKCREEK + +#include + +static int iRCCE_push_send_request(iRCCE_SEND_REQUEST *request) { + + char padline[RCCE_LINE_SIZE]; // copy buffer, used if message not multiple of line size + int test; // flag for calling iRCCE_test_flag() + + if(request->finished) return(iRCCE_SUCCESS); + + if(request->label == 1) goto label1; + if(request->label == 2) goto label2; + if(request->label == 3) goto label3; + + // send data in units of available chunk size of comm buffer + for (; request->wsize< (request->size / request->chunk) * request->chunk; request->wsize += request->chunk) { + request->bufptr = request->privbuf + request->wsize; + request->nbytes = request->chunk; + // copy private data to own comm buffer + iRCCE_put(request->combuf, (t_vcharp) request->bufptr, request->nbytes, RCCE_IAM); + RCCE_flag_write(request->sent, RCCE_FLAG_SET, request->dest); + // wait for the destination to be ready to receive a message +label1: + iRCCE_test_flag(*(request->ready), RCCE_FLAG_SET, &test); + if(!test) { + request->label = 1; + return(iRCCE_PENDING); + } + RCCE_flag_write(request->ready, RCCE_FLAG_UNSET, RCCE_IAM); + } + + request->remainder = request->size % request->chunk; + // if nothing is left over, we are done + if (!request->remainder) { + request->finished = 1; + return(iRCCE_SUCCESS); + } + + // send remainder of data--whole cache lines + request->bufptr = request->privbuf + (request->size / request->chunk) * request->chunk; + request->nbytes = request->remainder - request->remainder % RCCE_LINE_SIZE; + if (request->nbytes) { + // copy private data to own comm buffer + iRCCE_put(request->combuf, (t_vcharp)request->bufptr, request->nbytes, RCCE_IAM); + RCCE_flag_write(request->sent, RCCE_FLAG_SET, request->dest); + // wait for the destination to be ready to receive a message +label2: + iRCCE_test_flag(*(request->ready), RCCE_FLAG_SET, &test); + if(!test) { + request->label = 2; + return(iRCCE_PENDING); + } + RCCE_flag_write(request->ready, RCCE_FLAG_UNSET, RCCE_IAM); + } + + request->remainder = request->size % request->chunk; + request->remainder = request->remainder%RCCE_LINE_SIZE; + // if nothing is left over, we are done + if (!request->remainder) + { + request->finished = 1; + return(iRCCE_SUCCESS); + } + + // remainder is less than a cache line. This must be copied into appropriately sized + // intermediate space before it can be sent to the receiver + request->bufptr = request->privbuf + (request->size / request->chunk) * request->chunk + request->nbytes; + request->nbytes = RCCE_LINE_SIZE; + // copy private data to own comm buffer + memcpy(padline,request->bufptr,request->remainder); + iRCCE_put(request->combuf, (t_vcharp)padline, request->nbytes, RCCE_IAM); + RCCE_flag_write(request->sent, RCCE_FLAG_SET, request->dest); + // wait for the destination to be ready to receive a message +label3: + iRCCE_test_flag(*(request->ready), RCCE_FLAG_SET, &test); + if(!test) { + request->label = 3; + return(iRCCE_PENDING); + } + RCCE_flag_write(request->ready, RCCE_FLAG_UNSET, RCCE_IAM); + + request->finished = 1; + return(iRCCE_SUCCESS); +} + +static void iRCCE_init_send_request( + char *privbuf, // source buffer in local private memory (send buffer) + t_vcharp combuf, // intermediate buffer in MPB + size_t chunk, // size of MPB available for this message (bytes) + RCCE_FLAG *ready, // flag indicating whether receiver is ready + RCCE_FLAG *sent, // flag indicating whether message has been sent by source + size_t size, // size of message (bytes) + int dest, // UE that will receive the message + iRCCE_SEND_REQUEST *request + ) { + + request->privbuf = privbuf; + request->combuf = combuf; + request->chunk = chunk; + request->ready = ready; + request->sent = sent; + request->size = size; + request->dest = dest; + + request->wsize = 0; + request->remainder = 0; + request->nbytes = 0; + request->bufptr = NULL; + + request->label = 0; + + request->finished = 0; + + request->next = NULL; + + return; +} + +//-------------------------------------------------------------------------------------- +// FUNCTION: iRCCE_isend +//-------------------------------------------------------------------------------------- +// non-blocking send function; returns a handle of type iRCCE_SEND_REQUEST +//-------------------------------------------------------------------------------------- +static iRCCE_SEND_REQUEST blocking_isend_request; +int iRCCE_isend(char *privbuf, size_t size, int dest, iRCCE_SEND_REQUEST *request) { + + if(request == NULL) request = &blocking_isend_request; + + if (dest<0 || dest >= RCCE_NP) + return(RCCE_error_return(RCCE_debug_comm,RCCE_ERROR_ID)); + else { + iRCCE_init_send_request(privbuf, RCCE_buff_ptr, RCCE_chunk, + &RCCE_ready_flag[dest], &RCCE_sent_flag[RCCE_IAM], + size, dest, request); + + if(iRCCE_isend_queue == NULL) { + + if(iRCCE_push_send_request(request) == iRCCE_SUCCESS) { + return(iRCCE_SUCCESS); + } + else { + iRCCE_isend_queue = request; + + if(request == &blocking_isend_request) { + iRCCE_isend_wait(request); + return(iRCCE_SUCCESS); + } + + return(iRCCE_PENDING); + } + } + else { + if(iRCCE_isend_queue->next == NULL) { + iRCCE_isend_queue->next = request; + } + else { + iRCCE_SEND_REQUEST *run = iRCCE_isend_queue; + while(run->next != NULL) run = run->next; + run->next = request; + } + + if(request == &blocking_isend_request) { + iRCCE_isend_wait(request); + return(iRCCE_SUCCESS); + } + + return(iRCCE_RESERVED); + } + } +} + +//-------------------------------------------------------------------------------------- +// FUNCTION: iRCCE_isend_test +//-------------------------------------------------------------------------------------- +// test function for completion of the requestes non-blocking send operation +// Just provide NULL instead of testvar if you don't need it +//-------------------------------------------------------------------------------------- +int iRCCE_isend_test(iRCCE_SEND_REQUEST *request, int *test) { + + if(request == NULL) { + + iRCCE_isend_push(); + + if(iRCCE_isend_queue == NULL) { + if (test) (*test) = 1; + return(iRCCE_SUCCESS); + } + else { + if (test) (*test) = 0; + return(iRCCE_PENDING); + } + } + + if(request->finished) { + if (test) (*test) = 1; + return(iRCCE_SUCCESS); + } + + if(iRCCE_isend_queue != request) { + if (test) (*test) = 0; + return(iRCCE_RESERVED); + } + + iRCCE_push_send_request(request); + + if(request->finished) { + iRCCE_isend_queue = request->next; + + if (test) (*test) = 1; + return(iRCCE_SUCCESS); + } + + if (test) (*test) = 0; + return(iRCCE_PENDING); +} + + +//-------------------------------------------------------------------------------------- +// FUNCTION: iRCCE_isend_push +//-------------------------------------------------------------------------------------- +// progress function for pending requests in the isend queue +//-------------------------------------------------------------------------------------- +int iRCCE_isend_push(void) { + + iRCCE_SEND_REQUEST *request = iRCCE_isend_queue; + + if(request == NULL) { + return(iRCCE_SUCCESS); + } + + if(request->finished) { + return(iRCCE_SUCCESS); + } + + iRCCE_push_send_request(request); + + if(request->finished) { + iRCCE_isend_queue = request->next; + return(iRCCE_SUCCESS); + } + + return(iRCCE_PENDING); +} + + +//-------------------------------------------------------------------------------------- +// FUNCTION: iRCCE_isend_wait +//-------------------------------------------------------------------------------------- +// just wait for completion of the requestes non-blocking send operation +//-------------------------------------------------------------------------------------- +int iRCCE_isend_wait(iRCCE_SEND_REQUEST *request) { + + if(request != NULL) { + + while(!request->finished) { + + iRCCE_isend_push(); + iRCCE_irecv_push(); + } + } + else { + + while(iRCCE_isend_queue != NULL) { + + iRCCE_isend_push(); + iRCCE_irecv_push(); + } + } + + return(iRCCE_SUCCESS); +} + + +//-------------------------------------------------------------------------------------- +// FUNCTION: iRCCE_isend_cancel +//-------------------------------------------------------------------------------------- +// try to cancel a pending non-blocking send request +//-------------------------------------------------------------------------------------- +int iRCCE_isend_cancel(iRCCE_SEND_REQUEST *request, int *test) { + + iRCCE_SEND_REQUEST *run; + + if( (request == NULL) || (request->finished) ) { + if (test) (*test) = 0; + return iRCCE_NOT_ENQUEUED; + } + + if(iRCCE_isend_queue == NULL) { + if (test) (*test) = 0; + return iRCCE_NOT_ENQUEUED; + } + + if(iRCCE_isend_queue == request) { + if (test) (*test) = 0; + return iRCCE_PENDING; + } + + for(run = iRCCE_isend_queue; run->next != NULL; run = run->next) { + + // request found --> remove it from send queue: + if(run->next == request) { + + run->next = run->next->next; + + if (test) (*test) = 1; + return iRCCE_SUCCESS; + } + } + + if (test) (*test) = 0; + return iRCCE_NOT_ENQUEUED; +} + +#endif diff --git a/arch/x86/scc/iRCCE_put.c b/arch/x86/scc/iRCCE_put.c new file mode 100644 index 00000000..93cea070 --- /dev/null +++ b/arch/x86/scc/iRCCE_put.c @@ -0,0 +1,87 @@ +//*************************************************************************************** +// Put data into communication buffer. +//*************************************************************************************** +// +// Author: Rob F. Van der Wijngaart +// Intel Corporation +// Date: 008/30/2010 +// +//*************************************************************************************** +// +// Copyright 2010 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// [2010-11-03] switched to SCC-optimized memcpy() functions in scc_memcpy.h: +// - memcpy_to_mpb() +// - memcpy_from_mpb() +// by Stefan Lankes, Carsten Clauss, Chair for Operating Systems, +// RWTH Aachen University +// +#include +#include + +#ifdef CONFIG_ROCKCREEK + +#include + +#ifdef COPPERRIDGE +#include "scc_memcpy.h" +#endif + +void* iRCCE_memcpy_put(void *dest, const void *src, size_t count) +{ +#ifdef COPPERRIDGE + return memcpy_to_mpb(dest, src, count); +#else + return memcpy(dest, src, count); +#endif +} + +//-------------------------------------------------------------------------------------- +// FUNCTION: iRCCE_put +//-------------------------------------------------------------------------------------- +// copy data from address "source" in the local MPB or the calling UE's private memory +// to address "target" in the remote MPB. We do not test to see if a move from the +// calling UE's private memory stays within allocated memory +//-------------------------------------------------------------------------------------- +int iRCCE_put( + t_vcharp target, // target buffer, MPB + t_vcharp source, // source buffer, MPB or private memory + int num_bytes, + int ID + ) { + + // in non-GORY mode we only need to retain the MPB target shift; we + // already know the target is in the MPB, not private memory + target = RCCE_comm_buffer[ID]+(target-RCCE_comm_buffer[RCCE_IAM]); + + // make sure that any data that has been put in our MPB by another UE is visible +#ifdef _OPENMP + #pragma omp flush +#endif + + // do the actual copy + RC_cache_invalidate(); + + iRCCE_memcpy_put((void *)target, (void *)source, num_bytes); + + // flush data to make it visible to all threads; cannot use flush list because it + // concerns malloced space +#ifdef _OPENMP + #pragma omp flush +#endif + return(iRCCE_SUCCESS); +} + +#endif diff --git a/arch/x86/scc/iRCCE_recv.c b/arch/x86/scc/iRCCE_recv.c new file mode 100644 index 00000000..17beccfd --- /dev/null +++ b/arch/x86/scc/iRCCE_recv.c @@ -0,0 +1,190 @@ +//*************************************************************************************** +// Non-blocking receive routines. +//*************************************************************************************** +// +// Author: Rob F. Van der Wijngaart +// Intel Corporation +// Date: 008/30/2010 +// +//*************************************************************************************** +// +// Copyright 2010 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// [2010-11-26] added a _pipelined_ version of blocking send/recv +// by Carsten Clauss, Chair for Operating Systems, +// RWTH Aachen University +// +#include +#include + +#ifdef CONFIG_ROCKCREEK + +#include + +//-------------------------------------------------------------------------------------- +// FUNCTION: iRCCE_recv_general +//-------------------------------------------------------------------------------------- +// pipelined receive function +//-------------------------------------------------------------------------------------- +static int iRCCE_recv_general( + char *privbuf, // destination buffer in local private memory (receive buffer) + t_vcharp combuf, // intermediate buffer in MPB + size_t chunk, // size of MPB available for this message (bytes) + RCCE_FLAG *ready, // flag indicating whether receiver is ready + RCCE_FLAG *sent, // flag indicating whether message has been sent by source + size_t size, // size of message (bytes) + int source, // UE that sent the message + int *test // if 1 upon entry, do nonblocking receive; if message available + // set to 1, otherwise to 0 + ) { + + char padline[RCCE_LINE_SIZE]; // copy buffer, used if message not multiple of line size + size_t wsize, // offset within receive buffer when pulling in "chunk" bytes + remainder, // bytes remaining to be received + nbytes; // number of bytes to be received in single iRCCE_get call + int first_test; // only use first chunk to determine if message has been received yet + char *bufptr; // running pointer inside privbuf for current location + + first_test = 1; + +#if 0 + // receive data in units of available chunk size of MPB + for (wsize=0; wsize< (size/chunk)*chunk; wsize+=chunk) { + bufptr = privbuf + wsize; + nbytes = chunk; + // if function is called in test mode, check if first chunk has been sent already. + // If so, proceed as usual. If not, exit immediately + if (*test && first_test) { + first_test = 0; + if (!(*test = RCCE_probe(*sent))) return(iRCCE_SUCCESS); + } + RCCE_wait_until(*sent, RCCE_FLAG_SET); + RCCE_flag_write(sent, RCCE_FLAG_UNSET, RCCE_IAM); + // copy data from local MPB space to private memory + iRCCE_get((t_vcharp)bufptr, combuf, nbytes, source); + + // tell the source I have moved data out of its comm buffer + RCCE_flag_write(ready, RCCE_FLAG_SET, source); + } +#else + { // pipelined version of send/recv: + + size_t subchunk1 = chunk / 2; + size_t subchunk2 = chunk - subchunk1; + + for (wsize=0; wsize < (size/chunk)*chunk; wsize+=chunk) { + + if (*test && first_test) { + first_test = 0; + if (!(*test = RCCE_probe(*sent))) return(iRCCE_SUCCESS); + } + + bufptr = privbuf + wsize; + nbytes = subchunk1; + + RCCE_wait_until(*ready, RCCE_FLAG_SET); + RCCE_flag_write(ready, RCCE_FLAG_UNSET, RCCE_IAM); + iRCCE_get((t_vcharp)bufptr, combuf, nbytes, source); + + RCCE_flag_write(ready, RCCE_FLAG_SET, source); + + bufptr = privbuf + wsize + subchunk1; + nbytes = subchunk2; + + RCCE_wait_until(*sent, RCCE_FLAG_SET); + RCCE_flag_write(sent, RCCE_FLAG_UNSET, RCCE_IAM); + iRCCE_get((t_vcharp)bufptr, combuf + subchunk1, nbytes, source); + + RCCE_flag_write(sent, RCCE_FLAG_SET, source); + } + } +#endif + + remainder = size%chunk; + // if nothing is left over, we are done + if (!remainder) return(iRCCE_SUCCESS); + + // receive remainder of data--whole cache lines + bufptr = privbuf + (size/chunk)*chunk; + nbytes = remainder - remainder % RCCE_LINE_SIZE; + if (nbytes) { + // if function is called in test mode, check if first chunk has been sent already. + // If so, proceed as usual. If not, exit immediately + if (*test && first_test) { + first_test = 0; + if (!(*test = RCCE_probe(*sent))) return(iRCCE_SUCCESS); + } + RCCE_wait_until(*sent, RCCE_FLAG_SET); + RCCE_flag_write(sent, RCCE_FLAG_UNSET, RCCE_IAM); + // copy data from local MPB space to private memory + iRCCE_get((t_vcharp)bufptr, combuf, nbytes, source); + + // tell the source I have moved data out of its comm buffer + RCCE_flag_write(ready, RCCE_FLAG_SET, source); + } + + remainder = remainder % RCCE_LINE_SIZE; + if (!remainder) return(iRCCE_SUCCESS); + + // remainder is less than cache line. This must be copied into appropriately sized + // intermediate space before exact number of bytes get copied to the final destination + bufptr = privbuf + (size/chunk)*chunk + nbytes; + nbytes = RCCE_LINE_SIZE; + + // if function is called in test mode, check if first chunk has been sent already. + // If so, proceed as usual. If not, exit immediately + if (*test && first_test) { + first_test = 0; + if (!(*test = RCCE_probe(*sent))) return(iRCCE_SUCCESS); + } + RCCE_wait_until(*sent, RCCE_FLAG_SET); + RCCE_flag_write(sent, RCCE_FLAG_UNSET, RCCE_IAM); + + // copy data from local MPB space to private memory + iRCCE_get((t_vcharp)padline, combuf, nbytes, source); + memcpy(bufptr,padline,remainder); + + // tell the source I have moved data out of its comm buffer + RCCE_flag_write(ready, RCCE_FLAG_SET, source); + + return(iRCCE_SUCCESS); +} + + +//-------------------------------------------------------------------------------------- +// FUNCTION: iRCCE_recv +//-------------------------------------------------------------------------------------- +// pipelined recv function (blocking!) +//-------------------------------------------------------------------------------------- +int iRCCE_recv(char *privbuf, size_t size, int source) { + int ignore; + + while(iRCCE_irecv_queue[source] != NULL) { + // wait for completion of pending non-blocking requests + iRCCE_irecv_push(); + iRCCE_isend_push(); + } + + if (source<0 || source >= RCCE_NP) + return(RCCE_error_return(RCCE_debug_comm,RCCE_ERROR_ID)); + else { + ignore = 0; + return(iRCCE_recv_general(privbuf, RCCE_buff_ptr, RCCE_chunk, + &RCCE_ready_flag[RCCE_IAM], &RCCE_sent_flag[source], + size, source, &ignore)); + } +} + +#endif diff --git a/arch/x86/scc/iRCCE_send.c b/arch/x86/scc/iRCCE_send.c new file mode 100644 index 00000000..ad1582b3 --- /dev/null +++ b/arch/x86/scc/iRCCE_send.c @@ -0,0 +1,165 @@ +//*************************************************************************************** +// Synchronized receive routines. +//*************************************************************************************** +// +// Author: Rob F. Van der Wijngaart +// Intel Corporation +// Date: 008/30/2010 +// +//*************************************************************************************** +// +// Copyright 2010 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// [2010-11-26] added a _pipelined_ version of blocking send/recv +// by Carsten Clauss, Chair for Operating Systems, +// RWTH Aachen University +// +#include +#include + +#ifdef CONFIG_ROCKCREEK + +#include + +//-------------------------------------------------------------------------------------- +// FUNCTION: iRCCE_send_general +//-------------------------------------------------------------------------------------- +// pipelined send function +//-------------------------------------------------------------------------------------- +static int iRCCE_send_general( + char *privbuf, // source buffer in local private memory (send buffer) + t_vcharp combuf, // intermediate buffer in MPB + size_t chunk, // size of MPB available for this message (bytes) + RCCE_FLAG *ready, // flag indicating whether receiver is ready + RCCE_FLAG *sent, // flag indicating whether message has been sent by source + size_t size, // size of message (bytes) + int dest // UE that will receive the message + ) { + + char padline[RCCE_LINE_SIZE]; // copy buffer, used if message not multiple of line size + size_t wsize, // offset within send buffer when putting in "chunk" bytes + remainder, // bytes remaining to be sent + nbytes; // number of bytes to be sent in single iRCCE_put call + char *bufptr; // running pointer inside privbuf for current location + +#if 0 + // send data in units of available chunk size of comm buffer + for (wsize=0; wsize< (size/chunk)*chunk; wsize+=chunk) { + bufptr = privbuf + wsize; + nbytes = chunk; + // copy private data to own comm buffer + iRCCE_put(combuf, (t_vcharp) bufptr, nbytes, RCCE_IAM); + RCCE_flag_write(sent, RCCE_FLAG_SET, dest); + // wait for the destination to be ready to receive a message + RCCE_wait_until(*ready, RCCE_FLAG_SET); + RCCE_flag_write(ready, RCCE_FLAG_UNSET, RCCE_IAM); + } +#else + { // pipelined version of send/recv: + size_t subchunk1 = chunk / 2; + size_t subchunk2 = chunk - subchunk1; + + wsize = 0; + + for (; wsize < (size/chunk)*chunk; wsize+=chunk) { + + bufptr = privbuf + wsize; + nbytes = subchunk1; + + iRCCE_put(combuf, (t_vcharp) bufptr, nbytes, RCCE_IAM); + RCCE_flag_write(ready, RCCE_FLAG_SET, dest); + + if(wsize>0) + { + RCCE_wait_until(*sent, RCCE_FLAG_SET); + RCCE_flag_write(sent, RCCE_FLAG_UNSET, RCCE_IAM); + } + + bufptr = privbuf + wsize + subchunk1; + nbytes = subchunk2; + + iRCCE_put(combuf + subchunk1, (t_vcharp) bufptr, nbytes, RCCE_IAM); + RCCE_flag_write(sent, RCCE_FLAG_SET, dest); + + RCCE_wait_until(*ready, RCCE_FLAG_SET); + RCCE_flag_write(ready, RCCE_FLAG_UNSET, RCCE_IAM); + } + + if(wsize>0) { + RCCE_wait_until(*sent, RCCE_FLAG_SET); + RCCE_flag_write(sent, RCCE_FLAG_UNSET, RCCE_IAM); + } + } +#endif + + remainder = size%chunk; + // if nothing is left over, we are done + if (!remainder) return(iRCCE_SUCCESS); + + // send remainder of data--whole cache lines + bufptr = privbuf + (size/chunk)*chunk; + nbytes = remainder - remainder%RCCE_LINE_SIZE; + if (nbytes) { + // copy private data to own comm buffer + iRCCE_put(combuf, (t_vcharp)bufptr, nbytes, RCCE_IAM); + RCCE_flag_write(sent, RCCE_FLAG_SET, dest); + // wait for the destination to be ready to receive a message + RCCE_wait_until(*ready, RCCE_FLAG_SET); + RCCE_flag_write(ready, RCCE_FLAG_UNSET, RCCE_IAM); + } + + remainder = remainder%RCCE_LINE_SIZE; + if (!remainder) return(iRCCE_SUCCESS); + + // remainder is less than a cache line. This must be copied into appropriately sized + // intermediate space before it can be sent to the receiver + bufptr = privbuf + (size/chunk)*chunk + nbytes; + nbytes = RCCE_LINE_SIZE; + + // copy private data to own comm buffer + memcpy(padline,bufptr,remainder); + iRCCE_put(combuf, (t_vcharp)padline, nbytes, RCCE_IAM); + RCCE_flag_write(sent, RCCE_FLAG_SET, dest); + + // wait for the destination to be ready to receive a message + RCCE_wait_until(*ready, RCCE_FLAG_SET); + RCCE_flag_write(ready, RCCE_FLAG_UNSET, RCCE_IAM); + + return(iRCCE_SUCCESS); +} + + +//-------------------------------------------------------------------------------------- +// FUNCTION: iRCCE_send +//-------------------------------------------------------------------------------------- +// pipelined send function (blocking!) +//-------------------------------------------------------------------------------------- +int iRCCE_send(char *privbuf, size_t size, int dest) { + + while(iRCCE_isend_queue != NULL) { + // wait for completion of pending non-blocking requests + iRCCE_isend_push(); + iRCCE_irecv_push(); + } + + if (dest<0 || dest >= RCCE_NP) + return(RCCE_error_return(RCCE_debug_comm,RCCE_ERROR_ID)); + else + return(iRCCE_send_general(privbuf, RCCE_buff_ptr, RCCE_chunk, + &RCCE_ready_flag[dest], &RCCE_sent_flag[RCCE_IAM], + size, dest)); +} + +#endif diff --git a/arch/x86/scc/iRCCE_synch.c b/arch/x86/scc/iRCCE_synch.c new file mode 100644 index 00000000..5512728b --- /dev/null +++ b/arch/x86/scc/iRCCE_synch.c @@ -0,0 +1,127 @@ +///************************************************************************************* +// Synchronization functions. +// Single-bit and whole-cache-line flags are sufficiently different that we provide +// separate implementations of the synchronization routines for each case +//************************************************************************************** +// +// Author: Rob F. Van der Wijngaart +// Intel Corporation +// Date: 008/30/2010 +// +//************************************************************************************** +// +// Copyright 2010 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// [2010-10-25] added support for non-blocking send/recv operations +// - iRCCE_isend(), ..._test(), ..._wait(), ..._push() +// - iRCCE_irecv(), ..._test(), ..._wait(), ..._push() +// by Carsten Clauss, Chair for Operating Systems, +// RWTH Aachen University +// +// [2010-11-12] extracted non-blocking code into separate library +// by Carsten Scholtes +// +// [2011-01-21] updated the datatype of RCCE_FLAG according to the +// recent version of RCCE +// +// [2011-04-12] added marco test for rcce version +// +#include +#include + +#ifdef CONFIG_ROCKCREEK + +#include + +#ifdef SINGLEBITFLAGS + +int iRCCE_test_flag(RCCE_FLAG flag, RCCE_FLAG_STATUS val, int *result) { + + t_vcharp cflag; + +#ifdef RCCE_VERSION + // this is a newer version than V1.0.13 + t_vcharp flaga; +#endif + + cflag = flag.line_address; + +#ifdef RCCE_VERSION + // this is a newer version than V1.0.13 + flaga = flag.flag_addr; +#endif + + // always flush/invalidate to ensure we read the most recent value of *flag + // keep reading it until it has the required value + +#ifdef _OPENMP +#pragma omp flush +#endif + RC_cache_invalidate(); + +#ifdef RCCE_VERSION + // this is a newer version than V1.0.13 + if(RCCE_bit_value(flaga, (flag.location)%RCCE_FLAGS_PER_BYTE) != val) { +#else + if(RCCE_bit_value(cflag, flag.location) != val) { +#endif + (*result) = 0; + } + else { + (*result) = 1; + } + + return(iRCCE_SUCCESS); +} + +#else + +////////////////////////////////////////////////////////////////// +// LOCKLESS SYNCHRONIZATION USING ONE WHOLE CACHE LINE PER FLAG // +////////////////////////////////////////////////////////////////// + +int iRCCE_test_flag(RCCE_FLAG flag, RCCE_FLAG_STATUS val, int *result) { + +#ifdef RCCE_VERSION + // this is a newer version than V1.0.13 + t_vcharp flaga = flag.flag_addr; +#endif + + // always flush/invalidate to ensure we read the most recent value of *flag + // keep reading it until it has the required value. We only need to read the + // first int of the MPB cache line containing the flag +#ifdef _OPENMP +#pragma omp flush +#endif + RC_cache_invalidate(); + +#ifdef RCCE_VERSION + // this is a newer version than V1.0.13 + if((RCCE_FLAG_STATUS)(*flaga) != val) { +#else + if((*flag) != val) { +#endif + (*result) = 0; + } + else { + (*result) = 1; + } + + return(iRCCE_SUCCESS); +} + +#endif + +#endif