@@ -44,8 +44,8 @@ namespace copy_as_contig
44
44
45
45
template <typename T,
46
46
typename IndexerT,
47
- int vec_sz = 4 ,
48
- int n_vecs = 2 ,
47
+ std:: uint32_t vec_sz = 4u ,
48
+ std:: uint32_t n_vecs = 2u ,
49
49
bool enable_sg_loadstore = true >
50
50
class CopyAsCContigFunctor
51
51
{
@@ -66,53 +66,63 @@ class CopyAsCContigFunctor
66
66
67
67
void operator ()(sycl::nd_item<1 > ndit) const
68
68
{
69
+ static_assert (vec_sz > 0 );
70
+ static_assert (n_vecs > 0 );
71
+ static_assert (vec_sz * n_vecs < (std::uint32_t (1 ) << 8 ));
72
+
73
+ constexpr std::uint8_t elems_per_wi =
74
+ static_cast <std::uint8_t >(vec_sz * n_vecs);
75
+
69
76
using dpctl::tensor::type_utils::is_complex;
70
77
if constexpr (!enable_sg_loadstore || is_complex<T>::value) {
71
- const std::uint32_t sgSize =
78
+ const std::uint16_t sgSize =
72
79
ndit.get_sub_group ().get_local_range ()[0 ];
73
80
const std::size_t gid = ndit.get_global_linear_id ();
74
81
75
- const std::size_t base =
76
- (gid / sgSize) * sgSize * n_vecs * vec_sz + (gid % sgSize);
77
- for (size_t offset = base;
78
- offset < std::min (nelems, base + sgSize * (n_vecs * vec_sz));
79
- offset += sgSize)
80
- {
82
+ // base = (gid / sgSize) * sgSize * elems_per_wi + (gid % sgSize)
83
+ // gid % sgSize == gid - (gid / sgSize) * sgSize
84
+ const std::size_t elems_per_sg = sgSize * (elems_per_wi - 1 );
85
+ const std::size_t base = (gid / sgSize) * elems_per_sg + gid;
86
+ const std::size_t offset_max =
87
+ std::min (nelems, base + sgSize * elems_per_wi);
88
+
89
+ for (size_t offset = base; offset < offset_max; offset += sgSize) {
81
90
auto src_offset = src_indexer (offset);
82
91
dst_p[offset] = src_p[src_offset];
83
92
}
84
93
}
85
94
else {
86
95
auto sg = ndit.get_sub_group ();
87
- const std::uint32_t sgSize = sg.get_local_range ()[0 ];
88
- const size_t base = n_vecs * vec_sz *
89
- (ndit.get_group (0 ) * ndit.get_local_range (0 ) +
90
- sg.get_group_id ()[0 ] * sgSize);
96
+ const std::uint16_t sgSize = sg.get_max_local_range ()[0 ];
97
+ const size_t base =
98
+ elems_per_wi * (ndit.get_group (0 ) * ndit.get_local_range (0 ) +
99
+ sg.get_group_id ()[0 ] * sgSize);
91
100
92
- if (base + n_vecs * vec_sz * sgSize < nelems) {
101
+ if (base + elems_per_wi * sgSize < nelems) {
93
102
sycl::vec<T, vec_sz> dst_vec;
94
103
95
104
#pragma unroll
96
- for (std::uint32_t it = 0 ; it < n_vecs * vec_sz; it += vec_sz) {
105
+ for (std::uint8_t it = 0 ; it < elems_per_wi; it += vec_sz) {
106
+ const size_t block_start_id = base + it * sgSize;
97
107
auto dst_multi_ptr = sycl::address_space_cast<
98
108
sycl::access::address_space::global_space,
99
- sycl::access::decorated::yes>(
100
- &dst_p[base + it * sgSize]);
109
+ sycl::access::decorated::yes>(&dst_p[block_start_id]);
101
110
111
+ const size_t elem_id0 = block_start_id + sg.get_local_id ();
102
112
#pragma unroll
103
- for (std::uint32_t k = 0 ; k < vec_sz; k++) {
104
- ssize_t src_offset = src_indexer (
105
- base + (it + k) * sgSize + sg. get_local_id () );
113
+ for (std::uint8_t k = 0 ; k < vec_sz; k++) {
114
+ const size_t elem_id = elem_id0 + k * sgSize;
115
+ const ssize_t src_offset = src_indexer (elem_id );
106
116
dst_vec[k] = src_p[src_offset];
107
117
}
108
118
sg.store <vec_sz>(dst_multi_ptr, dst_vec);
109
119
}
110
120
}
111
121
else {
112
- for ( size_t k = base + sg.get_local_id ()[0 ]; k < nelems ;
113
- k += sgSize)
114
- {
115
- ssize_t src_offset = src_indexer (k);
122
+ const size_t lane_id = sg.get_local_id ()[0 ];
123
+ const size_t k0 = base + lane_id;
124
+ for ( size_t k = k0; k < nelems; k += sgSize) {
125
+ const ssize_t src_offset = src_indexer (k);
116
126
dst_p[k] = src_p[src_offset];
117
127
}
118
128
}
@@ -150,9 +160,11 @@ sycl::event submit_c_contiguous_copy(sycl::queue &exec_q,
150
160
const std::size_t lws =
151
161
((preferred_lws + max_sg_size - 1 ) / max_sg_size) * max_sg_size;
152
162
153
- constexpr std::uint32_t nelems_per_wi = n_vecs * vec_sz;
154
- size_t n_groups =
155
- (nelems + nelems_per_wi * lws - 1 ) / (nelems_per_wi * lws);
163
+ constexpr std::uint8_t nelems_per_wi = n_vecs * vec_sz;
164
+
165
+ const size_t nelems_per_group = nelems_per_wi * lws;
166
+ const size_t n_groups =
167
+ (nelems + nelems_per_group - 1 ) / (nelems_per_group);
156
168
157
169
sycl::event copy_ev = exec_q.submit ([&](sycl::handler &cgh) {
158
170
cgh.depends_on (depends);
@@ -171,8 +183,8 @@ sycl::event submit_c_contiguous_copy(sycl::queue &exec_q,
171
183
172
184
template <typename T,
173
185
typename IndexT,
174
- int vec_sz,
175
- int n_vecs,
186
+ std:: uint32_t vec_sz,
187
+ std:: uint32_t n_vecs,
176
188
bool enable_sgload>
177
189
class as_contig_krn ;
178
190
0 commit comments