Skip to content

Commit 04fd35c

Browse files
Tweaks to copy_as_contiguous CContig functor
1. Save common subexpressions to variables 2. Sub-group size type changed to uint16 (from uint32) 3. sg.get_local_range() replaced with sg.get_max_local_range() This is safe to do since work-group size is chosen to be a multiple of sub-group size for all possile choices of sub-group size (1, 8, 16, 32, 64) 4. Simplified computation of base value in generic branch for complex types, or when sg_load is disabled, to avoid a division (and left a comment)
1 parent a91fc80 commit 04fd35c

File tree

1 file changed

+41
-29
lines changed

1 file changed

+41
-29
lines changed

dpctl/tensor/libtensor/include/kernels/copy_as_contiguous.hpp

Lines changed: 41 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -44,8 +44,8 @@ namespace copy_as_contig
4444

4545
template <typename T,
4646
typename IndexerT,
47-
int vec_sz = 4,
48-
int n_vecs = 2,
47+
std::uint32_t vec_sz = 4u,
48+
std::uint32_t n_vecs = 2u,
4949
bool enable_sg_loadstore = true>
5050
class CopyAsCContigFunctor
5151
{
@@ -66,53 +66,63 @@ class CopyAsCContigFunctor
6666

6767
void operator()(sycl::nd_item<1> ndit) const
6868
{
69+
static_assert(vec_sz > 0);
70+
static_assert(n_vecs > 0);
71+
static_assert(vec_sz * n_vecs < (std::uint32_t(1) << 8));
72+
73+
constexpr std::uint8_t elems_per_wi =
74+
static_cast<std::uint8_t>(vec_sz * n_vecs);
75+
6976
using dpctl::tensor::type_utils::is_complex;
7077
if constexpr (!enable_sg_loadstore || is_complex<T>::value) {
71-
const std::uint32_t sgSize =
78+
const std::uint16_t sgSize =
7279
ndit.get_sub_group().get_local_range()[0];
7380
const std::size_t gid = ndit.get_global_linear_id();
7481

75-
const std::size_t base =
76-
(gid / sgSize) * sgSize * n_vecs * vec_sz + (gid % sgSize);
77-
for (size_t offset = base;
78-
offset < std::min(nelems, base + sgSize * (n_vecs * vec_sz));
79-
offset += sgSize)
80-
{
82+
// base = (gid / sgSize) * sgSize * elems_per_wi + (gid % sgSize)
83+
// gid % sgSize == gid - (gid / sgSize) * sgSize
84+
const std::size_t elems_per_sg = sgSize * (elems_per_wi - 1);
85+
const std::size_t base = (gid / sgSize) * elems_per_sg + gid;
86+
const std::size_t offset_max =
87+
std::min(nelems, base + sgSize * elems_per_wi);
88+
89+
for (size_t offset = base; offset < offset_max; offset += sgSize) {
8190
auto src_offset = src_indexer(offset);
8291
dst_p[offset] = src_p[src_offset];
8392
}
8493
}
8594
else {
8695
auto sg = ndit.get_sub_group();
87-
const std::uint32_t sgSize = sg.get_local_range()[0];
88-
const size_t base = n_vecs * vec_sz *
89-
(ndit.get_group(0) * ndit.get_local_range(0) +
90-
sg.get_group_id()[0] * sgSize);
96+
const std::uint16_t sgSize = sg.get_max_local_range()[0];
97+
const size_t base =
98+
elems_per_wi * (ndit.get_group(0) * ndit.get_local_range(0) +
99+
sg.get_group_id()[0] * sgSize);
91100

92-
if (base + n_vecs * vec_sz * sgSize < nelems) {
101+
if (base + elems_per_wi * sgSize < nelems) {
93102
sycl::vec<T, vec_sz> dst_vec;
94103

95104
#pragma unroll
96-
for (std::uint32_t it = 0; it < n_vecs * vec_sz; it += vec_sz) {
105+
for (std::uint8_t it = 0; it < elems_per_wi; it += vec_sz) {
106+
const size_t block_start_id = base + it * sgSize;
97107
auto dst_multi_ptr = sycl::address_space_cast<
98108
sycl::access::address_space::global_space,
99-
sycl::access::decorated::yes>(
100-
&dst_p[base + it * sgSize]);
109+
sycl::access::decorated::yes>(&dst_p[block_start_id]);
101110

111+
const size_t elem_id0 = block_start_id + sg.get_local_id();
102112
#pragma unroll
103-
for (std::uint32_t k = 0; k < vec_sz; k++) {
104-
ssize_t src_offset = src_indexer(
105-
base + (it + k) * sgSize + sg.get_local_id());
113+
for (std::uint8_t k = 0; k < vec_sz; k++) {
114+
const size_t elem_id = elem_id0 + k * sgSize;
115+
const ssize_t src_offset = src_indexer(elem_id);
106116
dst_vec[k] = src_p[src_offset];
107117
}
108118
sg.store<vec_sz>(dst_multi_ptr, dst_vec);
109119
}
110120
}
111121
else {
112-
for (size_t k = base + sg.get_local_id()[0]; k < nelems;
113-
k += sgSize)
114-
{
115-
ssize_t src_offset = src_indexer(k);
122+
const size_t lane_id = sg.get_local_id()[0];
123+
const size_t k0 = base + lane_id;
124+
for (size_t k = k0; k < nelems; k += sgSize) {
125+
const ssize_t src_offset = src_indexer(k);
116126
dst_p[k] = src_p[src_offset];
117127
}
118128
}
@@ -150,9 +160,11 @@ sycl::event submit_c_contiguous_copy(sycl::queue &exec_q,
150160
const std::size_t lws =
151161
((preferred_lws + max_sg_size - 1) / max_sg_size) * max_sg_size;
152162

153-
constexpr std::uint32_t nelems_per_wi = n_vecs * vec_sz;
154-
size_t n_groups =
155-
(nelems + nelems_per_wi * lws - 1) / (nelems_per_wi * lws);
163+
constexpr std::uint8_t nelems_per_wi = n_vecs * vec_sz;
164+
165+
const size_t nelems_per_group = nelems_per_wi * lws;
166+
const size_t n_groups =
167+
(nelems + nelems_per_group - 1) / (nelems_per_group);
156168

157169
sycl::event copy_ev = exec_q.submit([&](sycl::handler &cgh) {
158170
cgh.depends_on(depends);
@@ -171,8 +183,8 @@ sycl::event submit_c_contiguous_copy(sycl::queue &exec_q,
171183

172184
template <typename T,
173185
typename IndexT,
174-
int vec_sz,
175-
int n_vecs,
186+
std::uint32_t vec_sz,
187+
std::uint32_t n_vecs,
176188
bool enable_sgload>
177189
class as_contig_krn;
178190

0 commit comments

Comments
 (0)