28 #include <common/config.h> 41 #ifdef STARPU_HAVE_HWLOC 53 #ifdef STARPU_USE_MPI_MASTER_SLAVE 63 #define STARPU_MAX_PIPELINE 4 65 enum initialization { UNINITIALIZED = 0, CHANGING, INITIALIZED };
67 struct _starpu_ctx_change_list;
72 starpu_pthread_mutex_t mutex;
73 enum starpu_worker_archtype arch;
75 struct starpu_perfmodel_arch perf_arch;
95 #ifdef STARPU_SPINLOCK_CHECK 96 const char *relax_on_file;
98 const char *relax_on_func;
99 const char *relax_off_file;
101 const char *relax_off_func;
128 struct _starpu_ctx_change_list ctx_change_list;
129 struct starpu_task_list local_tasks;
135 struct starpu_task *current_tasks[STARPU_MAX_PIPELINE];
136 #ifdef STARPU_SIMGRID 137 starpu_pthread_wait_t wait;
140 struct timespec cl_start;
141 struct timespec cl_end;
147 unsigned worker_is_running;
148 unsigned worker_is_initialized;
163 unsigned removed_from_ctx[STARPU_NMAX_SCHED_CTXS+1];
175 unsigned shares_tasks_lists[STARPU_NMAX_SCHED_CTXS+1];
177 unsigned poped_in_ctx[STARPU_NMAX_SCHED_CTXS+1];
183 unsigned reverse_phase[2];
193 #ifdef STARPU_HAVE_HWLOC 194 hwloc_bitmap_t hwloc_cpu_set;
195 hwloc_obj_t hwloc_obj;
199 char padding[STARPU_CACHELINE_SIZE];
208 int combined_workerid[STARPU_NMAXWORKERS];
211 starpu_pthread_mutex_t count_mutex;
217 #ifdef STARPU_HAVE_HWLOC 218 hwloc_bitmap_t hwloc_cpu_set;
223 char padding[STARPU_CACHELINE_SIZE];
232 starpu_pthread_mutex_t mutex;
239 unsigned set_is_initialized;
242 #ifdef STARPU_USE_MPI_MASTER_SLAVE 254 unsigned nsched_ctxs;
256 #ifdef STARPU_HAVE_HWLOC 293 unsigned nworkerpercuda;
294 int cuda_th_per_stream;
302 unsigned nhwmpidevices;
304 unsigned nhwmpicores[STARPU_MAXMPIDEVS];
305 unsigned nmpicores[STARPU_MAXMPIDEVS];
310 unsigned nmicdevices;
312 unsigned nhwmiccores[STARPU_MAXMICDEVS];
313 unsigned nmiccores[STARPU_MAXMICDEVS];
322 unsigned workers_bindid[STARPU_NMAXWORKERS];
330 unsigned workers_cuda_gpuid[STARPU_NMAXWORKERS];
338 unsigned workers_opencl_gpuid[STARPU_NMAXWORKERS];
348 unsigned workers_mpi_ms_deviceid[STARPU_NMAXWORKERS];
355 #ifdef STARPU_HAVE_HWLOC 362 char currently_bound[STARPU_NMAXWORKERS];
363 char currently_shared[STARPU_NMAXWORKERS];
389 char padding1[STARPU_CACHELINE_SIZE];
399 starpu_pthread_mutex_t submitted_mutex;
402 char padding2[STARPU_CACHELINE_SIZE];
418 struct starpu_conf conf;
438 extern int _starpu_worker_parallel_blocks;
441 extern int _starpu_keys_initialized STARPU_ATTRIBUTE_INTERNAL;
442 extern starpu_pthread_key_t _starpu_worker_key STARPU_ATTRIBUTE_INTERNAL;
443 extern starpu_pthread_key_t _starpu_worker_set_key STARPU_ATTRIBUTE_INTERNAL;
447 int *_starpu_get_argc();
448 char ***_starpu_get_argv();
463 ANNOTATE_HAPPENS_AFTER(&_starpu_config.running);
464 ret = _starpu_config.running;
465 ANNOTATE_HAPPENS_BEFORE(&_starpu_config.running);
492 void _starpu_block_worker(
int workerid, starpu_pthread_cond_t *cond, starpu_pthread_mutex_t *mutex);
499 static inline unsigned _starpu_worker_get_count(
void)
501 return _starpu_config.topology.nworkers;
503 #define starpu_worker_get_count _starpu_worker_get_count 510 STARPU_ASSERT(_starpu_keys_initialized);
511 STARPU_PTHREAD_SETSPECIFIC(_starpu_worker_key, worker);
518 if (!_starpu_keys_initialized)
520 return (
struct _starpu_worker *) STARPU_PTHREAD_GETSPECIFIC(_starpu_worker_key);
528 STARPU_ASSERT(_starpu_keys_initialized);
529 STARPU_PTHREAD_SETSPECIFIC(_starpu_worker_set_key, worker);
536 if (!_starpu_keys_initialized)
538 return (
struct _starpu_worker_set *) STARPU_PTHREAD_GETSPECIFIC(_starpu_worker_set_key);
545 STARPU_ASSERT(
id < starpu_worker_get_count());
546 return &_starpu_config.workers[id];
553 return (
id > STARPU_NMAX_SCHED_CTXS) ? NULL : &_starpu_config.sched_ctxs[
id];
562 return &_starpu_config;
568 return _starpu_config.disable_kernels;
574 return _starpu_config.workers[workerid].status;
581 _starpu_config.workers[workerid].status = status;
587 return &_starpu_config.sched_ctxs[STARPU_GLOBAL_SCHED_CTX];
590 int starpu_worker_get_nids_by_type(
enum starpu_worker_archtype type,
int *workerids,
int maxsize);
596 static inline unsigned _starpu_worker_mutex_is_sched_mutex(
int workerid, starpu_pthread_mutex_t *mutex)
602 static inline int _starpu_worker_get_nsched_ctxs(
int workerid)
604 return _starpu_config.workers[
workerid].nsched_ctxs;
612 return _starpu_config.topology.nsched_ctxs;
632 #define starpu_worker_get_id _starpu_worker_get_id 640 int id = starpu_worker_get_id();
641 STARPU_ASSERT_MSG(
id>=0,
"%s:%d Cannot be called from outside a worker\n", f, l);
644 #define _starpu_worker_get_id_check(f,l) __starpu_worker_get_id_check(f,l) 646 enum starpu_node_kind _starpu_worker_get_node_kind(
enum starpu_worker_archtype type);
648 void _starpu_worker_set_stream_ctx(
unsigned workerid,
struct _starpu_sched_ctx *sched_ctx);
650 struct _starpu_sched_ctx* _starpu_worker_get_ctx_stream(
unsigned stream_workerid);
659 _starpu_worker_parallel_blocks = 1;
680 STARPU_PTHREAD_COND_BROADCAST(&worker->
sched_cond);
681 #ifdef STARPU_SIMGRID 682 starpu_pthread_queue_broadcast(&_starpu_simgrid_task_queue[worker->
workerid]);
698 STARPU_PTHREAD_COND_BROADCAST(&worker->
sched_cond);
727 STARPU_PTHREAD_COND_BROADCAST(&worker->
sched_cond);
741 STARPU_PTHREAD_COND_BROADCAST(&worker->
sched_cond);
770 STARPU_PTHREAD_COND_BROADCAST(&worker->
sched_cond);
787 STARPU_PTHREAD_COND_BROADCAST(&worker->
sched_cond);
807 #ifdef STARPU_SPINLOCK_CHECK 808 static inline void __starpu_worker_enter_sched_op(
struct _starpu_worker *
const worker,
const char*file,
int line,
const char* func)
846 #ifdef STARPU_SPINLOCK_CHECK 847 worker->relax_on_file = file;
848 worker->relax_on_line = line;
849 worker->relax_on_func = func;
852 #ifdef STARPU_SPINLOCK_CHECK 853 #define _starpu_worker_enter_sched_op(worker) __starpu_worker_enter_sched_op((worker), __FILE__, __LINE__, __starpu_func__) 861 #ifdef STARPU_SPINLOCK_CHECK 862 static inline void __starpu_worker_leave_sched_op(
struct _starpu_worker *
const worker,
const char*file,
int line,
const char* func)
864 static inline void _starpu_worker_leave_sched_op(
struct _starpu_worker *
const worker)
869 #ifdef STARPU_SPINLOCK_CHECK 870 worker->relax_off_file = file;
871 worker->relax_off_line = line;
872 worker->relax_off_func = func;
875 STARPU_PTHREAD_COND_BROADCAST(&worker->
sched_cond);
878 #ifdef STARPU_SPINLOCK_CHECK 879 #define _starpu_worker_leave_sched_op(worker) __starpu_worker_leave_sched_op((worker), __FILE__, __LINE__, __starpu_func__) 882 static inline int _starpu_worker_sched_op_pending(
void)
884 int workerid = starpu_worker_get_id();
888 STARPU_ASSERT(worker != NULL);
924 STARPU_PTHREAD_COND_BROADCAST(&worker->
sched_cond);
925 #ifdef STARPU_SIMGRID 926 starpu_pthread_queue_broadcast(&_starpu_simgrid_task_queue[worker->
workerid]);
947 STARPU_PTHREAD_COND_BROADCAST(&worker->
sched_cond);
952 #ifdef STARPU_SPINLOCK_CHECK 953 static inline void __starpu_worker_relax_on(
const char*file,
int line,
const char* func)
963 STARPU_PTHREAD_MUTEX_LOCK_SCHED(&worker->
sched_mutex);
964 #ifdef STARPU_SPINLOCK_CHECK 965 STARPU_ASSERT_MSG(worker->
state_relax_refcnt<UINT_MAX,
"relax last turn on in %s (%s:%d)\n", worker->relax_on_func, worker->relax_on_file, worker->relax_on_line);
970 #ifdef STARPU_SPINLOCK_CHECK 971 worker->relax_on_file = file;
972 worker->relax_on_line = line;
973 worker->relax_on_func = func;
975 STARPU_PTHREAD_COND_BROADCAST(&worker->
sched_cond);
976 STARPU_PTHREAD_MUTEX_UNLOCK_SCHED(&worker->
sched_mutex);
978 #ifdef STARPU_SPINLOCK_CHECK 979 #define _starpu_worker_relax_on() __starpu_worker_relax_on(__FILE__, __LINE__, __starpu_func__) 981 #define starpu_worker_relax_on _starpu_worker_relax_on 984 #ifdef STARPU_SPINLOCK_CHECK 985 static inline void __starpu_worker_relax_on_locked(
struct _starpu_worker *worker,
const char*file,
int line,
const char* func)
992 #ifdef STARPU_SPINLOCK_CHECK 993 STARPU_ASSERT_MSG(worker->
state_relax_refcnt<UINT_MAX,
"relax last turn on in %s (%s:%d)\n", worker->relax_on_func, worker->relax_on_file, worker->relax_on_line);
998 #ifdef STARPU_SPINLOCK_CHECK 999 worker->relax_on_file = file;
1000 worker->relax_on_line = line;
1001 worker->relax_on_func = func;
1003 STARPU_PTHREAD_COND_BROADCAST(&worker->
sched_cond);
1005 #ifdef STARPU_SPINLOCK_CHECK 1006 #define _starpu_worker_relax_on_locked(worker) __starpu_worker_relax_on_locked(worker,__FILE__, __LINE__, __starpu_func__) 1009 #ifdef STARPU_SPINLOCK_CHECK 1010 static inline void __starpu_worker_relax_off(
const char*file,
int line,
const char* func)
1012 static inline void _starpu_worker_relax_off(
void)
1015 int workerid = starpu_worker_get_id();
1019 STARPU_ASSERT(worker != NULL);
1022 STARPU_PTHREAD_MUTEX_LOCK_SCHED(&worker->
sched_mutex);
1023 #ifdef STARPU_SPINLOCK_CHECK 1024 STARPU_ASSERT_MSG(worker->
state_relax_refcnt>0,
"relax last turn off in %s (%s:%d)\n", worker->relax_on_func, worker->relax_on_file, worker->relax_on_line);
1029 #ifdef STARPU_SPINLOCK_CHECK 1030 worker->relax_off_file = file;
1031 worker->relax_off_line = line;
1032 worker->relax_off_func = func;
1034 STARPU_PTHREAD_MUTEX_UNLOCK_SCHED(&worker->
sched_mutex);
1036 #ifdef STARPU_SPINLOCK_CHECK 1037 #define _starpu_worker_relax_off() __starpu_worker_relax_off(__FILE__, __LINE__, __starpu_func__) 1039 #define starpu_worker_relax_off _starpu_worker_relax_off 1041 #ifdef STARPU_SPINLOCK_CHECK 1042 static inline void __starpu_worker_relax_off_locked(
const char*file,
int line,
const char* func)
1044 static inline void _starpu_worker_relax_off_locked(
void)
1047 int workerid = starpu_worker_get_id();
1051 STARPU_ASSERT(worker != NULL);
1054 #ifdef STARPU_SPINLOCK_CHECK 1055 STARPU_ASSERT_MSG(worker->
state_relax_refcnt>0,
"relax last turn off in %s (%s:%d)\n", worker->relax_on_func, worker->relax_on_file, worker->relax_on_line);
1060 #ifdef STARPU_SPINLOCK_CHECK 1061 worker->relax_off_file = file;
1062 worker->relax_off_line = line;
1063 worker->relax_off_func = func;
1066 #ifdef STARPU_SPINLOCK_CHECK 1067 #define _starpu_worker_relax_off_locked() __starpu_worker_relax_off_locked(__FILE__, __LINE__, __starpu_func__) 1070 static inline int _starpu_worker_get_relax_state(
void)
1072 int workerid = starpu_worker_get_id();
1076 STARPU_ASSERT(worker != NULL);
1079 #define starpu_worker_get_relax_state _starpu_worker_get_relax_state 1088 STARPU_ASSERT(worker != NULL);
1089 int cur_workerid = starpu_worker_get_id();
1090 if (workerid != cur_workerid)
1092 starpu_worker_relax_on();
1094 STARPU_PTHREAD_MUTEX_LOCK_SCHED(&worker->
sched_mutex);
1102 STARPU_PTHREAD_MUTEX_LOCK_SCHED(&worker->
sched_mutex);
1106 static inline int _starpu_worker_trylock(
int workerid)
1109 int cur_workerid = cur_worker->
workerid;
1111 STARPU_ASSERT(worker != NULL);
1114 int ret = STARPU_PTHREAD_MUTEX_TRYLOCK_SCHED(&cur_worker->
sched_mutex);
1117 if (workerid == cur_workerid)
1122 ret = STARPU_PTHREAD_MUTEX_TRYLOCK_SCHED(&worker->
sched_mutex);
1128 STARPU_PTHREAD_MUTEX_UNLOCK_SCHED(&worker->
sched_mutex);
1132 STARPU_PTHREAD_MUTEX_UNLOCK_SCHED(&cur_worker->
sched_mutex);
1136 static inline void _starpu_worker_unlock(
int workerid)
1139 STARPU_ASSERT(worker != NULL);
1140 STARPU_PTHREAD_MUTEX_UNLOCK_SCHED(&worker->
sched_mutex);
1141 int cur_workerid = starpu_worker_get_id();
1142 if (workerid != cur_workerid)
1144 starpu_worker_relax_off();
1148 static inline void _starpu_worker_lock_self(
void)
1150 int workerid = starpu_worker_get_id_check();
1152 STARPU_ASSERT(worker != NULL);
1153 STARPU_PTHREAD_MUTEX_LOCK_SCHED(&worker->
sched_mutex);
1156 static inline void _starpu_worker_unlock_self(
void)
1158 int workerid = starpu_worker_get_id_check();
1160 STARPU_ASSERT(worker != NULL);
1161 STARPU_PTHREAD_MUTEX_UNLOCK_SCHED(&worker->
sched_mutex);
1164 static inline int _starpu_wake_worker_relax(
int workerid)
1167 int ret = starpu_wake_worker_locked(workerid);
1168 _starpu_worker_unlock(workerid);
1172 int starpu_wake_worker_relax_light(
int workerid);
1182 #endif // __WORKERS_H__ struct starpu_task ** local_ordered_tasks
Definition: workers.h:130
void _starpu_may_pause(void)
unsigned devid
Definition: workers.h:77
static unsigned __starpu_worker_get_id_check(const char *f, int l)
Definition: workers.h:636
static void _starpu_worker_relax_on(void)
Definition: workers.h:955
int pause_depth
Definition: workers.h:427
unsigned char pipeline_length
Definition: workers.h:144
unsigned ncudagpus
Definition: workers.h:292
static void _starpu_worker_request_blocking_in_parallel(struct _starpu_worker *const worker)
Definition: workers.h:657
int workerid
Definition: workers.h:80
unsigned state_unblock_in_parallel_req
Definition: workers.h:110
unsigned current_ordered_task_order
Definition: workers.h:133
static unsigned _starpu_get_nsched_ctxs(void)
Definition: workers.h:608
unsigned subworkerid
Definition: workers.h:78
unsigned nhwcudagpus
Definition: workers.h:276
unsigned nb_buffers_totransfer
Definition: workers.h:168
int current_mpi_deviceid
Definition: workers.h:375
unsigned memory_node
Definition: workers.h:207
unsigned nhwopenclgpus
Definition: workers.h:281
void _starpu_worker_refuse_task(struct _starpu_worker *worker, struct starpu_task *task)
static void _starpu_worker_leave_changing_ctx_op(struct _starpu_worker *const worker)
Definition: workers.h:943
unsigned nmpidevices
Definition: workers.h:301
int combined_workerid
Definition: workers.h:81
Definition: workers.h:351
struct starpu_perfmodel_arch perf_arch
Definition: workers.h:204
unsigned has_prev_init
Definition: workers.h:161
unsigned run_by_starpu
Definition: workers.h:153
unsigned nsched_ctxs
Definition: workers.h:158
starpu_pthread_t thread_changing_ctx
Definition: workers.h:120
static struct _starpu_machine_config * _starpu_get_machine_config(void)
Definition: workers.h:560
void _starpu_set_argc_argv(int *argc, char ***argv)
uint32_t _starpu_can_submit_opencl_task(void)
unsigned started
Definition: workers.h:235
static void _starpu_set_local_worker_set_key(struct _starpu_worker_set *worker)
Definition: workers.h:526
int cuda_nodeid
Definition: workers.h:380
uint32_t worker_mask
Definition: workers.h:74
static struct _starpu_sched_ctx * _starpu_get_initial_sched_ctx(void)
Definition: workers.h:585
unsigned current_ordered_task
Definition: workers.h:132
unsigned local_ordered_tasks_size
Definition: workers.h:131
static void _starpu_worker_lock(int workerid)
Definition: workers.h:1085
static void _starpu_worker_request_unblocking_in_parallel(struct _starpu_worker *const worker)
Definition: workers.h:706
unsigned state_relax_refcnt
Definition: workers.h:94
struct starpu_task * task_transferring
Definition: workers.h:169
Definition: workers.h:246
unsigned pop_ctx_priority
Definition: workers.h:185
Definition: barrier_counter.h:26
static void _starpu_set_local_worker_key(struct _starpu_worker *worker)
Definition: workers.h:508
unsigned nhwmpi
Definition: workers.h:286
static struct _starpu_worker_set * _starpu_get_local_worker_set_key(void)
Definition: workers.h:534
unsigned block_in_parallel_ref_count
Definition: workers.h:119
unsigned submitting
Definition: workers.h:433
uint32_t _starpu_can_submit_cpu_task(void)
Definition: workers.h:202
uint32_t worker_mask
Definition: workers.h:205
unsigned id
Definition: sched_ctx.h:48
void _starpu_conf_check_environment(struct starpu_conf *conf)
starpu_pthread_cond_t ready_cond
Definition: workers.h:85
unsigned nb_buffers_transferred
Definition: workers.h:167
unsigned char first_task
Definition: workers.h:142
Definition: sched_ctx.h:45
unsigned memory_node
Definition: workers.h:86
unsigned nhwcpus
Definition: workers.h:266
unsigned nopenclgpus
Definition: workers.h:298
unsigned state_blocked_in_parallel
Definition: workers.h:106
struct starpu_conf conf
Definition: workers.h:418
static unsigned _starpu_machine_is_running(void)
Definition: workers.h:457
static enum _starpu_worker_status _starpu_worker_get_status(int workerid)
Definition: workers.h:572
unsigned state_block_in_parallel_ack
Definition: workers.h:109
unsigned nhwmicdevices
Definition: workers.h:309
int bindid
Definition: workers.h:79
static struct _starpu_worker * _starpu_get_worker_struct(unsigned id)
Definition: workers.h:543
starpu_pthread_cond_t ready_cond
Definition: workers.h:238
unsigned state_changing_ctx_waiting
Definition: workers.h:104
int current_bindid
Definition: workers.h:361
static void _starpu_worker_enter_changing_ctx_op(struct _starpu_worker *const worker)
Definition: workers.h:901
unsigned ncpus
Definition: workers.h:289
void _starpu_block_worker(int workerid, starpu_pthread_cond_t *cond, starpu_pthread_mutex_t *mutex)
int current_mic_deviceid
Definition: workers.h:372
static void _starpu_worker_process_block_in_parallel_requests(struct _starpu_worker *const worker)
Definition: workers.h:755
static void _starpu_worker_relax_on_locked(struct _starpu_worker *worker)
Definition: workers.h:987
unsigned spinning_backoff
Definition: workers.h:165
int current_cuda_gpuid
Definition: workers.h:366
void _starpu_driver_start(struct _starpu_worker *worker, unsigned fut_key, unsigned sync)
static void _starpu_worker_enter_sched_op(struct _starpu_worker *const worker)
Definition: workers.h:810
struct starpu_tree * tree
Definition: workers.h:261
int current_rank
Definition: workers.h:82
Definition: sched_ctx_list.h:24
unsigned numa_memory_node
Definition: workers.h:87
uint32_t worker_mask
Definition: workers.h:415
unsigned state_sched_op_pending
Definition: workers.h:103
starpu_pthread_t worker_thread
Definition: workers.h:76
int worker_size
Definition: workers.h:83
unsigned ncombinedworkers
Definition: workers.h:252
unsigned nworkers
Definition: workers.h:249
unsigned is_slave_somewhere
Definition: workers.h:186
Definition: workers.h:230
int mic_nodeid
Definition: workers.h:384
unsigned state_block_in_parallel_req
Definition: workers.h:108
uint32_t _starpu_can_submit_cuda_task(void)
uint32_t _starpu_worker_exists(struct starpu_task *)
void _starpu_worker_apply_deferred_ctx_changes(void)
unsigned char pipeline_stuck
Definition: workers.h:145
hwloc_topology_t hwtopology
Definition: workers.h:258
static void _starpu_worker_set_status(int workerid, enum _starpu_worker_status status)
Definition: workers.h:579
unsigned state_changing_ctx_notice
Definition: workers.h:105
int cpus_nodeid
Definition: workers.h:378
unsigned state_unblock_in_parallel_ack
Definition: workers.h:111
struct starpu_task * current_task
Definition: workers.h:134
starpu_pthread_cond_t sched_cond
Definition: workers.h:92
starpu_pthread_mutex_t sched_mutex
Definition: workers.h:93
static int _starpu_get_disable_kernels(void)
Definition: workers.h:566
unsigned _starpu_worker_can_block(unsigned memnode, struct _starpu_worker *worker)
unsigned nbindid
Definition: workers.h:410
static struct _starpu_worker * _starpu_get_local_worker_key(void)
Definition: workers.h:516
unsigned state_blocked_in_parallel_observed
Definition: workers.h:107
static int _starpu_worker_get_id(void)
Definition: workers.h:616
int opencl_nodeid
Definition: workers.h:382
void _starpu_worker_start(struct _starpu_worker *worker, unsigned fut_key, unsigned sync)
unsigned running
Definition: workers.h:421
starpu_pthread_cond_t started_cond
Definition: workers.h:84
int starpu_worker_get_nids_ctx_free_by_type(enum starpu_worker_archtype type, int *workerids, int maxsize)
int current_opencl_gpuid
Definition: workers.h:369
void _starpu_worker_init(struct _starpu_worker *workerarg, struct _starpu_machine_config *pconfig)
_starpu_worker_status
Definition: errorcheck.h:25
static struct _starpu_sched_ctx * _starpu_get_sched_ctx_struct(unsigned id)
Definition: workers.h:551
unsigned char ntasks
Definition: workers.h:143
unsigned state_keep_awake
Definition: workers.h:150
unsigned nhwpus
Definition: workers.h:271
int mpi_nodeid
Definition: workers.h:386
starpu_pthread_t worker_thread
Definition: workers.h:233