@@ -44,8 +44,8 @@ namespace copy_as_contig
4444
4545template <typename T,
4646 typename IndexerT,
47- int vec_sz = 4 ,
48- int n_vecs = 2 ,
47+ std:: uint32_t vec_sz = 4u ,
48+ std:: uint32_t n_vecs = 2u ,
4949 bool enable_sg_loadstore = true >
5050class CopyAsCContigFunctor
5151{
@@ -66,53 +66,63 @@ class CopyAsCContigFunctor
6666
6767 void operator ()(sycl::nd_item<1 > ndit) const
6868 {
69+ static_assert (vec_sz > 0 );
70+ static_assert (n_vecs > 0 );
71+ static_assert (vec_sz * n_vecs < (std::uint32_t (1 ) << 8 ));
72+
73+ constexpr std::uint8_t elems_per_wi =
74+ static_cast <std::uint8_t >(vec_sz * n_vecs);
75+
6976 using dpctl::tensor::type_utils::is_complex;
7077 if constexpr (!enable_sg_loadstore || is_complex<T>::value) {
71- const std::uint32_t sgSize =
78+ const std::uint16_t sgSize =
7279 ndit.get_sub_group ().get_local_range ()[0 ];
7380 const std::size_t gid = ndit.get_global_linear_id ();
7481
75- const std::size_t base =
76- (gid / sgSize) * sgSize * n_vecs * vec_sz + (gid % sgSize);
77- for (size_t offset = base;
78- offset < std::min (nelems, base + sgSize * (n_vecs * vec_sz));
79- offset += sgSize)
80- {
82+ // base = (gid / sgSize) * sgSize * elems_per_wi + (gid % sgSize)
83+ // gid % sgSize == gid - (gid / sgSize) * sgSize
84+ const std::size_t elems_per_sg = sgSize * (elems_per_wi - 1 );
85+ const std::size_t base = (gid / sgSize) * elems_per_sg + gid;
86+ const std::size_t offset_max =
87+ std::min (nelems, base + sgSize * elems_per_wi);
88+
89+ for (size_t offset = base; offset < offset_max; offset += sgSize) {
8190 auto src_offset = src_indexer (offset);
8291 dst_p[offset] = src_p[src_offset];
8392 }
8493 }
8594 else {
8695 auto sg = ndit.get_sub_group ();
87- const std::uint32_t sgSize = sg.get_local_range ()[0 ];
88- const size_t base = n_vecs * vec_sz *
89- (ndit.get_group (0 ) * ndit.get_local_range (0 ) +
90- sg.get_group_id ()[0 ] * sgSize);
96+ const std::uint16_t sgSize = sg.get_max_local_range ()[0 ];
97+ const size_t base =
98+ elems_per_wi * (ndit.get_group (0 ) * ndit.get_local_range (0 ) +
99+ sg.get_group_id ()[0 ] * sgSize);
91100
92- if (base + n_vecs * vec_sz * sgSize < nelems) {
101+ if (base + elems_per_wi * sgSize < nelems) {
93102 sycl::vec<T, vec_sz> dst_vec;
94103
95104#pragma unroll
96- for (std::uint32_t it = 0 ; it < n_vecs * vec_sz; it += vec_sz) {
105+ for (std::uint8_t it = 0 ; it < elems_per_wi; it += vec_sz) {
106+ const size_t block_start_id = base + it * sgSize;
97107 auto dst_multi_ptr = sycl::address_space_cast<
98108 sycl::access::address_space::global_space,
99- sycl::access::decorated::yes>(
100- &dst_p[base + it * sgSize]);
109+ sycl::access::decorated::yes>(&dst_p[block_start_id]);
101110
111+ const size_t elem_id0 = block_start_id + sg.get_local_id ();
102112#pragma unroll
103- for (std::uint32_t k = 0 ; k < vec_sz; k++) {
104- ssize_t src_offset = src_indexer (
105- base + (it + k) * sgSize + sg. get_local_id () );
113+ for (std::uint8_t k = 0 ; k < vec_sz; k++) {
114+ const size_t elem_id = elem_id0 + k * sgSize;
115+ const ssize_t src_offset = src_indexer (elem_id );
106116 dst_vec[k] = src_p[src_offset];
107117 }
108118 sg.store <vec_sz>(dst_multi_ptr, dst_vec);
109119 }
110120 }
111121 else {
112- for ( size_t k = base + sg.get_local_id ()[0 ]; k < nelems ;
113- k += sgSize)
114- {
115- ssize_t src_offset = src_indexer (k);
122+ const size_t lane_id = sg.get_local_id ()[0 ];
123+ const size_t k0 = base + lane_id;
124+ for ( size_t k = k0; k < nelems; k += sgSize) {
125+ const ssize_t src_offset = src_indexer (k);
116126 dst_p[k] = src_p[src_offset];
117127 }
118128 }
@@ -150,9 +160,11 @@ sycl::event submit_c_contiguous_copy(sycl::queue &exec_q,
150160 const std::size_t lws =
151161 ((preferred_lws + max_sg_size - 1 ) / max_sg_size) * max_sg_size;
152162
153- constexpr std::uint32_t nelems_per_wi = n_vecs * vec_sz;
154- size_t n_groups =
155- (nelems + nelems_per_wi * lws - 1 ) / (nelems_per_wi * lws);
163+ constexpr std::uint8_t nelems_per_wi = n_vecs * vec_sz;
164+
165+ const size_t nelems_per_group = nelems_per_wi * lws;
166+ const size_t n_groups =
167+ (nelems + nelems_per_group - 1 ) / (nelems_per_group);
156168
157169 sycl::event copy_ev = exec_q.submit ([&](sycl::handler &cgh) {
158170 cgh.depends_on (depends);
@@ -171,8 +183,8 @@ sycl::event submit_c_contiguous_copy(sycl::queue &exec_q,
171183
172184template <typename T,
173185 typename IndexT,
174- int vec_sz,
175- int n_vecs,
186+ std:: uint32_t vec_sz,
187+ std:: uint32_t n_vecs,
176188 bool enable_sgload>
177189class as_contig_krn ;
178190
0 commit comments