Skip to content

Non-destructive multi-gpu defaults #99

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 9 commits into
base: master
Choose a base branch
from
Draft

Non-destructive multi-gpu defaults #99

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
d62c277
wip on new multi-gpu support
JBlaschke Feb 9, 2021
877f9a2
fix precompiler bug
JBlaschke Feb 9, 2021
46698a5
fix typo
JBlaschke Feb 10, 2021
4acc0af
cleanup device logic, next: propagate to rest
JBlaschke Feb 14, 2021
b3dca86
wrap setpts
JBlaschke Feb 14, 2021
c500bba
more wrappers
JBlaschke Feb 14, 2021
3dc6f85
fix recursion, and forgotten set device
JBlaschke Feb 14, 2021
04f96bf
wrap memtransfer function
JBlaschke Feb 14, 2021
912c257
update defaults
JBlaschke Feb 14, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,13 +80,15 @@ STATICLIB=lib-static/$(LIBNAME).a
BINDIR=bin

HEADERS = include/cufinufft.h src/cudeconvolve.h src/memtransfer.h include/profile.h \
src/cuspreadinterp.h include/cufinufft_eitherprec.h include/cufinufft_errors.h
src/cuspreadinterp.h include/cufinufft_eitherprec.h include/cufinufft_errors.h \
include/multi_gpu_policy.h
CONTRIBOBJS=contrib/dirft2d.o contrib/common.o contrib/spreadinterp.o contrib/utils_fp.o

# We create three collections of objects:
# Double (_64), Single (_32), and floating point agnostic (no suffix)

CUFINUFFTOBJS=src/precision_independent.o src/profile.o contrib/legendre_rule_fast.o contrib/utils.o
CUFINUFFTOBJS=src/precision_independent.o src/profile.o contrib/legendre_rule_fast.o \
contrib/utils.o src/multi_gpu_policy.o
CUFINUFFTOBJS_64=src/2d/spreadinterp2d.o src/2d/cufinufft2d.o \
src/2d/spread2d_wrapper.o src/2d/spread2d_wrapper_paul.o \
src/2d/interp2d_wrapper.o src/memtransfer_wrapper.o \
Expand Down
37 changes: 37 additions & 0 deletions include/cufinufft_eitherprec.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,9 +30,13 @@
/* Undefine things so we don't get warnings/errors later */
#undef CUFINUFFT_DEFAULT_OPTS
#undef CUFINUFFT_MAKEPLAN
#undef __CUFINUFFT_MAKEPLAN
#undef CUFINUFFT_SETPTS
#undef __CUFINUFFT_SETPTS
#undef CUFINUFFT_EXECUTE
#undef __CUFINUFFT_EXECUTE
#undef CUFINUFFT_DESTROY
#undef __CUFINUFFT_DESTROY
#undef CUFINUFFT2D1_EXEC
#undef CUFINUFFT2D2_EXEC
#undef CUFINUFFT3D1_EXEC
Expand All @@ -43,11 +47,17 @@
#undef ALLOCGPUMEM1D_NUPTS
#undef FREEGPUMEMORY1D
#undef ALLOCGPUMEM2D_PLAN
#undef __ALLOCGPUMEM2D_PLAN
#undef ALLOCGPUMEM2D_NUPTS
#undef __ALLOCGPUMEM2D_NUPTS
#undef FREEGPUMEMORY2D
#undef __FREEGPUMEMORY2D
#undef ALLOCGPUMEM3D_PLAN
#undef __ALLOCGPUMEM3D_PLAN
#undef ALLOCGPUMEM3D_NUPTS
#undef __ALLOCGPUMEM3D_NUPTS
#undef FREEGPUMEMORY3D
#undef __FREEGPUMEMORY3D
/* spreading and interp only*/
#undef CUFINUFFT_SPREAD2D
#undef CUFINUFFT_SPREAD3D
Expand Down Expand Up @@ -88,9 +98,13 @@

#define CUFINUFFT_DEFAULT_OPTS cufinufftf_default_opts
#define CUFINUFFT_MAKEPLAN cufinufftf_makeplan
#define __CUFINUFFT_MAKEPLAN __cufinufftf_makeplan
#define CUFINUFFT_SETPTS cufinufftf_setpts
#define __CUFINUFFT_SETPTS __cufinufftf_setpts
#define CUFINUFFT_EXECUTE cufinufftf_execute
#define __CUFINUFFT_EXECUTE __cufinufftf_execute
#define CUFINUFFT_DESTROY cufinufftf_destroy
#define __CUFINUFFT_DESTROY __cufinufftf_destroy
#define CUFINUFFT2D1_EXEC cufinufftf2d1_exec
#define CUFINUFFT2D2_EXEC cufinufftf2d2_exec
#define CUFINUFFT3D1_EXEC cufinufftf3d1_exec
Expand All @@ -101,11 +115,17 @@
#define ALLOCGPUMEM1D_NUPTS allocgpumem1df_nupts
#define FREEGPUMEMORY1D freegpumemory1df
#define ALLOCGPUMEM2D_PLAN allocgpumem2df_plan
#define __ALLOCGPUMEM2D_PLAN __allocgpumem2df_plan
#define ALLOCGPUMEM2D_NUPTS allocgpumem2df_nupts
#define __ALLOCGPUMEM2D_NUPTS __allocgpumem2df_nupts
#define FREEGPUMEMORY2D freegpumemory2df
#define __FREEGPUMEMORY2D __freegpumemory2df
#define ALLOCGPUMEM3D_PLAN allocgpumem3df_plan
#define __ALLOCGPUMEM3D_PLAN __allocgpumem3df_plan
#define ALLOCGPUMEM3D_NUPTS allocgpumem3df_nupts
#define __ALLOCGPUMEM3D_NUPTS __allocgpumem3df_nupts
#define FREEGPUMEMORY3D freegpumemory3df
#define __FREEGPUMEMORY3D __freegpumemory3df
/* spreading and interp only*/
#define CUFINUFFT_SPREAD2D cufinufft_spread2df
#define CUFINUFFT_SPREAD3D cufinufft_spread3df
Expand Down Expand Up @@ -145,9 +165,13 @@

#define CUFINUFFT_DEFAULT_OPTS cufinufft_default_opts
#define CUFINUFFT_MAKEPLAN cufinufft_makeplan
#define __CUFINUFFT_MAKEPLAN __cufinufft_makeplan
#define CUFINUFFT_SETPTS cufinufft_setpts
#define __CUFINUFFT_SETPTS __cufinufft_setpts
#define CUFINUFFT_EXECUTE cufinufft_execute
#define __CUFINUFFT_EXECUTE __cufinufft_execute
#define CUFINUFFT_DESTROY cufinufft_destroy
#define __CUFINUFFT_DESTROY __cufinufft_destroy
#define CUFINUFFT2D1_EXEC cufinufft2d1_exec
#define CUFINUFFT2D2_EXEC cufinufft2d2_exec
#define CUFINUFFT3D1_EXEC cufinufft3d1_exec
Expand All @@ -158,11 +182,17 @@
#define ALLOCGPUMEM1D_NUPTS allocgpumem1d_nupts
#define FREEGPUMEMORY1D freegpumemory1d
#define ALLOCGPUMEM2D_PLAN allocgpumem2d_plan
#define __ALLOCGPUMEM2D_PLAN __allocgpumem2d_plan
#define ALLOCGPUMEM2D_NUPTS allocgpumem2d_nupts
#define __ALLOCGPUMEM2D_NUPTS __allocgpumem2d_nupts
#define FREEGPUMEMORY2D freegpumemory2d
#define __FREEGPUMEMORY2D __freegpumemory2d
#define ALLOCGPUMEM3D_PLAN allocgpumem3d_plan
#define __ALLOCGPUMEM3D_PLAN __allocgpumem3d_plan
#define ALLOCGPUMEM3D_NUPTS allocgpumem3d_nupts
#define __ALLOCGPUMEM3D_NUPTS __allocgpumem3d_nupts
#define FREEGPUMEMORY3D freegpumemory3d
#define __FREEGPUMEMORY3D __freegpumemory3d
/* spreading and interp only*/
#define CUFINUFFT_SPREAD2D cufinufft_spread2d
#define CUFINUFFT_SPREAD3D cufinufft_spread3d
Expand Down Expand Up @@ -271,10 +301,17 @@ int CUFINUFFT_DEFAULT_OPTS(int type, int dim, cufinufft_opts *opts);
int CUFINUFFT_MAKEPLAN(int type, int dim, int *n_modes, int iflag,
int ntransf, FLT tol, int maxbatchsize,
CUFINUFFT_PLAN *d_plan_ptr, cufinufft_opts *opts);
int __CUFINUFFT_MAKEPLAN(int type, int dim, int *n_modes, int iflag,
int ntransf, FLT tol, int maxbatchsize,
CUFINUFFT_PLAN *d_plan_ptr, cufinufft_opts *opts);
int CUFINUFFT_SETPTS(int M, FLT* h_kx, FLT* h_ky, FLT* h_kz, int N, FLT *h_s,
FLT *h_t, FLT *h_u, CUFINUFFT_PLAN d_plan);
int __CUFINUFFT_SETPTS(int M, FLT* h_kx, FLT* h_ky, FLT* h_kz, int N, FLT *h_s,
FLT *h_t, FLT *h_u, CUFINUFFT_PLAN d_plan);
int CUFINUFFT_EXECUTE(CUCPX* h_c, CUCPX* h_fk, CUFINUFFT_PLAN d_plan);
int __CUFINUFFT_EXECUTE(CUCPX* h_c, CUCPX* h_fk, CUFINUFFT_PLAN d_plan);
int CUFINUFFT_DESTROY(CUFINUFFT_PLAN d_plan);
int __CUFINUFFT_DESTROY(CUFINUFFT_PLAN d_plan);
#ifdef __cplusplus
}
#endif
Expand Down
2 changes: 2 additions & 0 deletions include/cufinufft_opts.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@ typedef struct cufinufft_opts { // see cufinufft_default_opts() for defaults

/* multi-gpu support */
int gpu_device_id;
int gpu_primary_ctx;
int gpu_force_primary_ctx;
} cufinufft_opts;

#endif
21 changes: 21 additions & 0 deletions include/multi_gpu_policy.h
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
#ifndef __MULTI_GPU_POLICY_H__
#define __MULTI_GPU_POLICY_H__

#include <cuda.h>
#include <cufinufft_opts.h>



typedef struct CtxProfile {
int i_dev;
int is_primary;
int is_clean;
} CtxProfile;



CUresult get_current_device(CtxProfile * ctx_profile);
int use_set_device(CtxProfile * ctx_profile, cufinufft_opts * opts);
int policy_set_device(cufinufft_opts * opts);

#endif
4 changes: 3 additions & 1 deletion python/cufinufft/_cufinufft.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,9 @@ def _get_NufftOpts():
('gpu_nstreams', c_int),
('gpu_kerevalmeth', c_int),
('gpu_spreadinterponly', c_int),
('gpu_device_id', c_int)]
('gpu_device_id', c_int),
('gpu_primary_ctx', c_int),
('gpu_force_primary_ctx', c_int)]
return fields


Expand Down
Loading