From 5d0dabe19c73e2f68c0088c9309738fe94f2b805 Mon Sep 17 00:00:00 2001 From: lshzh-ww Date: Mon, 24 Jul 2023 01:30:01 -0400 Subject: [PATCH 1/3] metal: concurrently dispatch commands Function `ggml_metal_graph_find_concurrency` will run and write commands that can be issued concurrently to metal context `concur_list` array, when `ggml_metal_graph_compute` is called for the first time. --- ggml-metal.m | 132 +++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 113 insertions(+), 19 deletions(-) diff --git a/ggml-metal.m b/ggml-metal.m index bf3f68fe45726..ff4f2b23a726a 100644 --- a/ggml-metal.m +++ b/ggml-metal.m @@ -36,6 +36,9 @@ int n_buffers; struct ggml_metal_buffer buffers[GGML_METAL_MAX_BUFFERS]; + int concur_list[GGML_MAX_NODES]; + int concur_list_len; + // custom kernels #define GGML_METAL_DECL_KERNEL(name) \ id function_##name; \ @@ -98,6 +101,7 @@ @implementation GGMLMetalClass ctx->device = MTLCreateSystemDefaultDevice(); ctx->queue = [ctx->device newCommandQueue]; ctx->n_buffers = 0; + ctx->concur_list_len = 0; // determine if we can use MPS if (MPSSupportsMTLDevice(ctx->device)) { @@ -355,11 +359,92 @@ void ggml_metal_get_tensor( memcpy(t->data, (void *) ((uint8_t *) id_src.contents + offs), ggml_nbytes(t)); } +void ggml_metal_graph_find_concurrency( + struct ggml_metal_context * ctx, + struct ggml_cgraph * gf) { + int search_depth = gf->n_nodes; //we only find concurrency in this range to avoid wasting too much time + int nodes_unused[GGML_MAX_NODES]; + + for (int i = 0; i < GGML_MAX_NODES; i++) {ctx->concur_list[i] = 0;} + for (int i = 0; i < gf->n_nodes; i++) {nodes_unused[i] = 1;} + ctx->concur_list_len = 0; + + int n_left = gf->n_nodes; + int n_start = 0; // all nodes before n_start at nodes_unused array have been sorted and store back to ctx->concur_list + int level_pos = 0; // at ctx->concur_list, the last layer (level) ends at level_pos + + while (n_left > 0) { + // number of nodes at a layer (that can be issued concurrently) + int concurrency = 0; + for (int i = n_start; i < ((n_start + search_depth > gf->n_nodes) ? gf->n_nodes : n_start + search_depth); i++) { + if (nodes_unused[i]) { + // if the requirements for gf->nodes[i] are satisfied + int exe_flag=1; + // scan all srcs + for (int src_ind = 0; src_ind < GGML_MAX_SRC; src_ind++) { + struct ggml_tensor * src_cur = gf->nodes[i]->src[src_ind]; + if (src_cur) { + // if is leaf nodes it's satisfied. + if (src_cur->op == GGML_OP_NONE && src_cur->grad == NULL) {continue;} + + // otherwise this src should be the output from previous nodes. + int is_found = 0; + // scan 2*search_depth back because we inserted barrier. + for (int j = ((level_pos - 2*search_depth) < 0 ? 0 : (level_pos - 2*search_depth)); j < level_pos; j++) { + if (gf->nodes[ctx->concur_list[j]] == src_cur) {is_found = 1; break;} + } + if (is_found == 0) {exe_flag = 0; break;} + } + } + if (exe_flag) { + // check if nodes[i]'s data will be overwritten by a node before nodes[i]. + // if node[5] and node[3] write to the same memory region, then we can't issue node[5] before node[3] + int64_t data_start = (int64_t) gf->nodes[i]->data; + int64_t length = (int64_t) ggml_nbytes(gf->nodes[i]); + for (int j = n_start; j < i; j++) { + if (nodes_unused[j] && gf->nodes[j]->op != GGML_OP_RESHAPE \ + && gf->nodes[j]->op != GGML_OP_VIEW \ + && gf->nodes[j]->op != GGML_OP_TRANSPOSE \ + && gf->nodes[j]->op != GGML_OP_PERMUTE) { + if (((int64_t)gf->nodes[j]->data) >= data_start + length || \ + ((int64_t)gf->nodes[j]->data) + (int64_t) ggml_nbytes(gf->nodes[j]) <= data_start) { + continue; + } else { + exe_flag = 0; + } + } + } + } + if (exe_flag) { + ctx->concur_list[level_pos + concurrency] = i; + nodes_unused[i] = 0; + concurrency++; + ctx->concur_list_len++; + } + } + } + n_left -= concurrency; + // adding a barrier different layer + ctx->concur_list[level_pos + concurrency] = -1; + ctx->concur_list_len++; + // jump all sorted nodes at nodes_bak + while (!nodes_unused[n_start]) {n_start++;} + level_pos += concurrency + 1; + } + + if (ctx->concur_list_len > GGML_MAX_NODES) { + fprintf(stderr, "%s: too many elements for metal ctx->concur_list!\n", __func__); + } +} + void ggml_metal_graph_compute( struct ggml_metal_context * ctx, struct ggml_cgraph * gf) { metal_printf("%s: evaluating graph\n", __func__); + if (!ctx->concur_list_len) { + ggml_metal_graph_find_concurrency(ctx,gf); + } // create multiple command buffers and enqueue them // then, we encode the graph into the command buffers in parallel @@ -378,7 +463,7 @@ void ggml_metal_graph_compute( dispatch_queue_t queue = dispatch_queue_create("llama.cpp", DISPATCH_QUEUE_CONCURRENT); for (int cb_idx = 0; cb_idx < n_cb; ++cb_idx) { - const int n_nodes_per_cb = (gf->n_nodes + n_cb - 1) / n_cb; + const int n_nodes_per_cb = (ctx->concur_list_len + n_cb - 1) / n_cb; dispatch_async(queue, ^{ size_t offs_src0 = 0; @@ -390,9 +475,18 @@ void ggml_metal_graph_compute( id encoder = nil; const int node_start = (cb_idx + 0) * n_nodes_per_cb; - const int node_end = (cb_idx == n_cb - 1) ? gf->n_nodes : (cb_idx + 1) * n_nodes_per_cb; - - for (int i = node_start; i < node_end; ++i) { + const int node_end = (cb_idx == n_cb - 1) ? ctx->concur_list_len : (cb_idx + 1) * n_nodes_per_cb; + + for (int ind = node_start; ind < node_end; ++ind) { + int i = ctx->concur_list[ind]; + if (i == -1) { + if (encoder == nil) { + encoder = [command_buffer computeCommandEncoderWithDispatchType: MTLDispatchTypeConcurrent]; + continue; + } + [encoder memoryBarrierWithScope:MTLBarrierScopeBuffers]; + continue; + } metal_printf("%s: encoding node %3d, op = %8s\n", __func__, i, ggml_op_name(gf->nodes[i]->op)); struct ggml_tensor * src0 = gf->nodes[i]->src[0]; @@ -463,7 +557,7 @@ void ggml_metal_graph_compute( case GGML_OP_ADD: { if (encoder == nil) { - encoder = [command_buffer computeCommandEncoder]; + encoder = [command_buffer computeCommandEncoderWithDispatchType: MTLDispatchTypeConcurrent]; } if (ggml_nelements(src1) == ne10) { @@ -484,7 +578,7 @@ void ggml_metal_graph_compute( case GGML_OP_MUL: { if (encoder == nil) { - encoder = [command_buffer computeCommandEncoder]; + encoder = [command_buffer computeCommandEncoderWithDispatchType: MTLDispatchTypeConcurrent]; } if (ggml_nelements(src1) == ne10) { @@ -505,7 +599,7 @@ void ggml_metal_graph_compute( case GGML_OP_SCALE: { if (encoder == nil) { - encoder = [command_buffer computeCommandEncoder]; + encoder = [command_buffer computeCommandEncoderWithDispatchType: MTLDispatchTypeConcurrent]; } const float scale = *(const float *) src1->data; @@ -522,7 +616,7 @@ void ggml_metal_graph_compute( case GGML_OP_SILU: { if (encoder == nil) { - encoder = [command_buffer computeCommandEncoder]; + encoder = [command_buffer computeCommandEncoderWithDispatchType: MTLDispatchTypeConcurrent]; } [encoder setComputePipelineState:ctx->pipeline_silu]; @@ -536,7 +630,7 @@ void ggml_metal_graph_compute( case GGML_OP_RELU: { if (encoder == nil) { - encoder = [command_buffer computeCommandEncoder]; + encoder = [command_buffer computeCommandEncoderWithDispatchType: MTLDispatchTypeConcurrent]; } [encoder setComputePipelineState:ctx->pipeline_relu]; @@ -550,7 +644,7 @@ void ggml_metal_graph_compute( case GGML_OP_GELU: { if (encoder == nil) { - encoder = [command_buffer computeCommandEncoder]; + encoder = [command_buffer computeCommandEncoderWithDispatchType: MTLDispatchTypeConcurrent]; } [encoder setComputePipelineState:ctx->pipeline_gelu]; @@ -564,7 +658,7 @@ void ggml_metal_graph_compute( case GGML_OP_SOFT_MAX: { if (encoder == nil) { - encoder = [command_buffer computeCommandEncoder]; + encoder = [command_buffer computeCommandEncoderWithDispatchType: MTLDispatchTypeConcurrent]; } const int nth = 32; @@ -582,7 +676,7 @@ void ggml_metal_graph_compute( case GGML_OP_DIAG_MASK_INF: { if (encoder == nil) { - encoder = [command_buffer computeCommandEncoder]; + encoder = [command_buffer computeCommandEncoderWithDispatchType: MTLDispatchTypeConcurrent]; } const int n_past = ((int32_t *)(dst->op_params))[0]; @@ -645,7 +739,7 @@ void ggml_metal_graph_compute( } } else { if (encoder == nil) { - encoder = [command_buffer computeCommandEncoder]; + encoder = [command_buffer computeCommandEncoderWithDispatchType: MTLDispatchTypeConcurrent]; } int nth0 = 32; @@ -772,7 +866,7 @@ void ggml_metal_graph_compute( case GGML_OP_GET_ROWS: { if (encoder == nil) { - encoder = [command_buffer computeCommandEncoder]; + encoder = [command_buffer computeCommandEncoderWithDispatchType: MTLDispatchTypeConcurrent]; } switch (src0->type) { @@ -801,7 +895,7 @@ void ggml_metal_graph_compute( case GGML_OP_RMS_NORM: { if (encoder == nil) { - encoder = [command_buffer computeCommandEncoder]; + encoder = [command_buffer computeCommandEncoderWithDispatchType: MTLDispatchTypeConcurrent]; } const float eps = 1e-6f; @@ -823,7 +917,7 @@ void ggml_metal_graph_compute( case GGML_OP_NORM: { if (encoder == nil) { - encoder = [command_buffer computeCommandEncoder]; + encoder = [command_buffer computeCommandEncoderWithDispatchType: MTLDispatchTypeConcurrent]; } const float eps = 1e-5f; @@ -845,7 +939,7 @@ void ggml_metal_graph_compute( case GGML_OP_ALIBI: { if (encoder == nil) { - encoder = [command_buffer computeCommandEncoder]; + encoder = [command_buffer computeCommandEncoderWithDispatchType: MTLDispatchTypeConcurrent]; } GGML_ASSERT((src0t == GGML_TYPE_F32)); @@ -888,7 +982,7 @@ void ggml_metal_graph_compute( case GGML_OP_ROPE: { if (encoder == nil) { - encoder = [command_buffer computeCommandEncoder]; + encoder = [command_buffer computeCommandEncoderWithDispatchType: MTLDispatchTypeConcurrent]; } const int n_past = ((int32_t *) dst->op_params)[0]; @@ -932,7 +1026,7 @@ void ggml_metal_graph_compute( case GGML_OP_CONT: { if (encoder == nil) { - encoder = [command_buffer computeCommandEncoder]; + encoder = [command_buffer computeCommandEncoderWithDispatchType: MTLDispatchTypeConcurrent]; } const int nth = 32; From 6f489a77dd399fb32fbe1ac734eaa61e6bc2d900 Mon Sep 17 00:00:00 2001 From: lshzh-ww Date: Mon, 24 Jul 2023 11:59:12 -0400 Subject: [PATCH 2/3] metal: don't call find_concurrency automatically. --- ggml-metal.h | 7 ++++++ ggml-metal.m | 60 +++++++++++++++++++++++++++++++++------------------- llama.cpp | 3 +++ 3 files changed, 48 insertions(+), 22 deletions(-) diff --git a/ggml-metal.h b/ggml-metal.h index 928f1705c381c..16f1a0caacfac 100644 --- a/ggml-metal.h +++ b/ggml-metal.h @@ -61,6 +61,13 @@ void ggml_metal_set_tensor(struct ggml_metal_context * ctx, struct ggml_tensor * // get data from the device into host memory void ggml_metal_get_tensor(struct ggml_metal_context * ctx, struct ggml_tensor * t); +// try to find operations that can be run concurrently in the graph +// you should run it again if the topology of your graph changes +void ggml_metal_graph_find_concurrency(struct ggml_metal_context * ctx, struct ggml_cgraph * gf); + +// if the graph has been optimized for concurrently dispatch +bool ggml_metal_if_optimized(struct ggml_metal_context * ctx); + // same as ggml_graph_compute but uses Metal // creates gf->n_threads command buffers in parallel void ggml_metal_graph_compute(struct ggml_metal_context * ctx, struct ggml_cgraph * gf); diff --git a/ggml-metal.m b/ggml-metal.m index ff4f2b23a726a..80e435e01d586 100644 --- a/ggml-metal.m +++ b/ggml-metal.m @@ -221,6 +221,13 @@ void ggml_metal_set_n_cb(struct ggml_metal_context * ctx, int n_cb) { ctx->n_cb = n_cb; } +bool ggml_metal_if_optimized(struct ggml_metal_context * ctx) { + if (ctx->concur_list_len) { + return true; + } + return false; +} + // finds the Metal buffer that contains the tensor data on the GPU device // the assumption is that there is 1-to-1 mapping between the host and device memory buffers, so we can find the // Metal buffer based on the host memory pointer @@ -441,9 +448,15 @@ void ggml_metal_graph_compute( struct ggml_metal_context * ctx, struct ggml_cgraph * gf) { metal_printf("%s: evaluating graph\n", __func__); - - if (!ctx->concur_list_len) { - ggml_metal_graph_find_concurrency(ctx,gf); + + // if there is ctx->concur_list, dispatch concurrently + // else fallback to serial dispatch + MTLComputePassDescriptor * encoder_descriptor = MTLComputePassDescriptor.computePassDescriptor; + encoder_descriptor.dispatchType = MTLDispatchTypeSerial; + int all_nodes_len = gf->n_nodes; + if (ctx->concur_list_len) { + encoder_descriptor.dispatchType = MTLDispatchTypeConcurrent; + all_nodes_len = ctx->concur_list_len; } // create multiple command buffers and enqueue them // then, we encode the graph into the command buffers in parallel @@ -463,7 +476,7 @@ void ggml_metal_graph_compute( dispatch_queue_t queue = dispatch_queue_create("llama.cpp", DISPATCH_QUEUE_CONCURRENT); for (int cb_idx = 0; cb_idx < n_cb; ++cb_idx) { - const int n_nodes_per_cb = (ctx->concur_list_len + n_cb - 1) / n_cb; + const int n_nodes_per_cb = (all_nodes_len + n_cb - 1) / n_cb; dispatch_async(queue, ^{ size_t offs_src0 = 0; @@ -475,13 +488,16 @@ void ggml_metal_graph_compute( id encoder = nil; const int node_start = (cb_idx + 0) * n_nodes_per_cb; - const int node_end = (cb_idx == n_cb - 1) ? ctx->concur_list_len : (cb_idx + 1) * n_nodes_per_cb; + const int node_end = (cb_idx == n_cb - 1) ? all_nodes_len : (cb_idx + 1) * n_nodes_per_cb; for (int ind = node_start; ind < node_end; ++ind) { - int i = ctx->concur_list[ind]; + int i = ind; + if (ctx->concur_list_len) { + i = ctx->concur_list[ind]; + } if (i == -1) { if (encoder == nil) { - encoder = [command_buffer computeCommandEncoderWithDispatchType: MTLDispatchTypeConcurrent]; + encoder = [command_buffer computeCommandEncoderWithDescriptor: encoder_descriptor]; continue; } [encoder memoryBarrierWithScope:MTLBarrierScopeBuffers]; @@ -557,7 +573,7 @@ void ggml_metal_graph_compute( case GGML_OP_ADD: { if (encoder == nil) { - encoder = [command_buffer computeCommandEncoderWithDispatchType: MTLDispatchTypeConcurrent]; + encoder = [command_buffer computeCommandEncoderWithDescriptor: encoder_descriptor]; } if (ggml_nelements(src1) == ne10) { @@ -578,7 +594,7 @@ void ggml_metal_graph_compute( case GGML_OP_MUL: { if (encoder == nil) { - encoder = [command_buffer computeCommandEncoderWithDispatchType: MTLDispatchTypeConcurrent]; + encoder = [command_buffer computeCommandEncoderWithDescriptor: encoder_descriptor]; } if (ggml_nelements(src1) == ne10) { @@ -599,7 +615,7 @@ void ggml_metal_graph_compute( case GGML_OP_SCALE: { if (encoder == nil) { - encoder = [command_buffer computeCommandEncoderWithDispatchType: MTLDispatchTypeConcurrent]; + encoder = [command_buffer computeCommandEncoderWithDescriptor: encoder_descriptor]; } const float scale = *(const float *) src1->data; @@ -616,7 +632,7 @@ void ggml_metal_graph_compute( case GGML_OP_SILU: { if (encoder == nil) { - encoder = [command_buffer computeCommandEncoderWithDispatchType: MTLDispatchTypeConcurrent]; + encoder = [command_buffer computeCommandEncoderWithDescriptor: encoder_descriptor]; } [encoder setComputePipelineState:ctx->pipeline_silu]; @@ -630,7 +646,7 @@ void ggml_metal_graph_compute( case GGML_OP_RELU: { if (encoder == nil) { - encoder = [command_buffer computeCommandEncoderWithDispatchType: MTLDispatchTypeConcurrent]; + encoder = [command_buffer computeCommandEncoderWithDescriptor: encoder_descriptor]; } [encoder setComputePipelineState:ctx->pipeline_relu]; @@ -644,7 +660,7 @@ void ggml_metal_graph_compute( case GGML_OP_GELU: { if (encoder == nil) { - encoder = [command_buffer computeCommandEncoderWithDispatchType: MTLDispatchTypeConcurrent]; + encoder = [command_buffer computeCommandEncoderWithDescriptor: encoder_descriptor]; } [encoder setComputePipelineState:ctx->pipeline_gelu]; @@ -658,7 +674,7 @@ void ggml_metal_graph_compute( case GGML_OP_SOFT_MAX: { if (encoder == nil) { - encoder = [command_buffer computeCommandEncoderWithDispatchType: MTLDispatchTypeConcurrent]; + encoder = [command_buffer computeCommandEncoderWithDescriptor: encoder_descriptor]; } const int nth = 32; @@ -676,7 +692,7 @@ void ggml_metal_graph_compute( case GGML_OP_DIAG_MASK_INF: { if (encoder == nil) { - encoder = [command_buffer computeCommandEncoderWithDispatchType: MTLDispatchTypeConcurrent]; + encoder = [command_buffer computeCommandEncoderWithDescriptor: encoder_descriptor]; } const int n_past = ((int32_t *)(dst->op_params))[0]; @@ -739,7 +755,7 @@ void ggml_metal_graph_compute( } } else { if (encoder == nil) { - encoder = [command_buffer computeCommandEncoderWithDispatchType: MTLDispatchTypeConcurrent]; + encoder = [command_buffer computeCommandEncoderWithDescriptor: encoder_descriptor]; } int nth0 = 32; @@ -866,7 +882,7 @@ void ggml_metal_graph_compute( case GGML_OP_GET_ROWS: { if (encoder == nil) { - encoder = [command_buffer computeCommandEncoderWithDispatchType: MTLDispatchTypeConcurrent]; + encoder = [command_buffer computeCommandEncoderWithDescriptor: encoder_descriptor]; } switch (src0->type) { @@ -895,7 +911,7 @@ void ggml_metal_graph_compute( case GGML_OP_RMS_NORM: { if (encoder == nil) { - encoder = [command_buffer computeCommandEncoderWithDispatchType: MTLDispatchTypeConcurrent]; + encoder = [command_buffer computeCommandEncoderWithDescriptor: encoder_descriptor]; } const float eps = 1e-6f; @@ -917,7 +933,7 @@ void ggml_metal_graph_compute( case GGML_OP_NORM: { if (encoder == nil) { - encoder = [command_buffer computeCommandEncoderWithDispatchType: MTLDispatchTypeConcurrent]; + encoder = [command_buffer computeCommandEncoderWithDescriptor: encoder_descriptor]; } const float eps = 1e-5f; @@ -939,7 +955,7 @@ void ggml_metal_graph_compute( case GGML_OP_ALIBI: { if (encoder == nil) { - encoder = [command_buffer computeCommandEncoderWithDispatchType: MTLDispatchTypeConcurrent]; + encoder = [command_buffer computeCommandEncoderWithDescriptor: encoder_descriptor]; } GGML_ASSERT((src0t == GGML_TYPE_F32)); @@ -982,7 +998,7 @@ void ggml_metal_graph_compute( case GGML_OP_ROPE: { if (encoder == nil) { - encoder = [command_buffer computeCommandEncoderWithDispatchType: MTLDispatchTypeConcurrent]; + encoder = [command_buffer computeCommandEncoderWithDescriptor: encoder_descriptor]; } const int n_past = ((int32_t *) dst->op_params)[0]; @@ -1026,7 +1042,7 @@ void ggml_metal_graph_compute( case GGML_OP_CONT: { if (encoder == nil) { - encoder = [command_buffer computeCommandEncoderWithDispatchType: MTLDispatchTypeConcurrent]; + encoder = [command_buffer computeCommandEncoderWithDescriptor: encoder_descriptor]; } const int nth = 32; diff --git a/llama.cpp b/llama.cpp index 5a8453bec3ada..04f8eab9184ff 100644 --- a/llama.cpp +++ b/llama.cpp @@ -1710,6 +1710,9 @@ static bool llama_eval_internal( #ifdef GGML_USE_METAL if (lctx.ctx_metal && N == 1) { + if (!ggml_metal_if_optimized(lctx.ctx_metal)) { + ggml_metal_graph_find_concurrency(lctx.ctx_metal,&gf); + } ggml_metal_set_n_cb (lctx.ctx_metal, n_threads); ggml_metal_graph_compute(lctx.ctx_metal, &gf); ggml_metal_get_tensor (lctx.ctx_metal, cur); From 141d88d91602e736fd0103a917cd940f6bccc241 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Tue, 25 Jul 2023 14:59:17 +0300 Subject: [PATCH 3/3] metal : code style changes --- ggml-metal.m | 59 ++++++++++++++++++++++++++-------------------------- 1 file changed, 29 insertions(+), 30 deletions(-) diff --git a/ggml-metal.m b/ggml-metal.m index e73fb2517608b..74a6bff404117 100644 --- a/ggml-metal.m +++ b/ggml-metal.m @@ -451,13 +451,13 @@ void ggml_metal_graph_compute( // if there is ctx->concur_list, dispatch concurrently // else fallback to serial dispatch - MTLComputePassDescriptor * encoder_descriptor = MTLComputePassDescriptor.computePassDescriptor; - encoder_descriptor.dispatchType = MTLDispatchTypeSerial; - int all_nodes_len = gf->n_nodes; - if (ctx->concur_list_len) { - encoder_descriptor.dispatchType = MTLDispatchTypeConcurrent; - all_nodes_len = ctx->concur_list_len; - } + MTLComputePassDescriptor * edesc = MTLComputePassDescriptor.computePassDescriptor; + + const bool has_concur = ctx->concur_list_len && ctx->concur_list_len <= GGML_MAX_NODES; + + const int n_nodes = has_concur ? ctx->concur_list_len : gf->n_nodes; + edesc.dispatchType = has_concur ? MTLDispatchTypeConcurrent : MTLDispatchTypeSerial; + // create multiple command buffers and enqueue them // then, we encode the graph into the command buffers in parallel @@ -476,7 +476,7 @@ void ggml_metal_graph_compute( dispatch_queue_t queue = dispatch_queue_create("llama.cpp", DISPATCH_QUEUE_CONCURRENT); for (int cb_idx = 0; cb_idx < n_cb; ++cb_idx) { - const int n_nodes_per_cb = (all_nodes_len + n_cb - 1) / n_cb; + const int n_nodes_per_cb = (n_nodes + n_cb - 1) / n_cb; dispatch_async(queue, ^{ size_t offs_src0 = 0; @@ -487,22 +487,21 @@ void ggml_metal_graph_compute( id encoder = nil; - const int node_start = (cb_idx + 0) * n_nodes_per_cb; - const int node_end = (cb_idx == n_cb - 1) ? all_nodes_len : (cb_idx + 1) * n_nodes_per_cb; + const int node_start = (cb_idx + 0) * n_nodes_per_cb; + const int node_end = (cb_idx == n_cb - 1) ? n_nodes : (cb_idx + 1) * n_nodes_per_cb; for (int ind = node_start; ind < node_end; ++ind) { - int i = ind; - if (ctx->concur_list_len) { - i = ctx->concur_list[ind]; - } + const int i = has_concur ? ctx->concur_list[ind] : ind; + if (i == -1) { if (encoder == nil) { - encoder = [command_buffer computeCommandEncoderWithDescriptor: encoder_descriptor]; + encoder = [command_buffer computeCommandEncoderWithDescriptor: edesc]; continue; } [encoder memoryBarrierWithScope:MTLBarrierScopeBuffers]; continue; } + metal_printf("%s: encoding node %3d, op = %8s\n", __func__, i, ggml_op_name(gf->nodes[i]->op)); struct ggml_tensor * src0 = gf->nodes[i]->src[0]; @@ -573,7 +572,7 @@ void ggml_metal_graph_compute( case GGML_OP_ADD: { if (encoder == nil) { - encoder = [command_buffer computeCommandEncoderWithDescriptor: encoder_descriptor]; + encoder = [command_buffer computeCommandEncoderWithDescriptor: edesc]; } if (ggml_nelements(src1) == ne10) { @@ -594,7 +593,7 @@ void ggml_metal_graph_compute( case GGML_OP_MUL: { if (encoder == nil) { - encoder = [command_buffer computeCommandEncoderWithDescriptor: encoder_descriptor]; + encoder = [command_buffer computeCommandEncoderWithDescriptor: edesc]; } if (ggml_nelements(src1) == ne10) { @@ -615,7 +614,7 @@ void ggml_metal_graph_compute( case GGML_OP_SCALE: { if (encoder == nil) { - encoder = [command_buffer computeCommandEncoderWithDescriptor: encoder_descriptor]; + encoder = [command_buffer computeCommandEncoderWithDescriptor: edesc]; } const float scale = *(const float *) src1->data; @@ -634,7 +633,7 @@ void ggml_metal_graph_compute( case GGML_UNARY_OP_SILU: { if (encoder == nil) { - encoder = [command_buffer computeCommandEncoderWithDescriptor: encoder_descriptor]; + encoder = [command_buffer computeCommandEncoderWithDescriptor: edesc]; } [encoder setComputePipelineState:ctx->pipeline_silu]; @@ -648,7 +647,7 @@ void ggml_metal_graph_compute( case GGML_UNARY_OP_RELU: { if (encoder == nil) { - encoder = [command_buffer computeCommandEncoderWithDescriptor: encoder_descriptor]; + encoder = [command_buffer computeCommandEncoderWithDescriptor: edesc]; } [encoder setComputePipelineState:ctx->pipeline_relu]; @@ -662,7 +661,7 @@ void ggml_metal_graph_compute( case GGML_UNARY_OP_GELU: { if (encoder == nil) { - encoder = [command_buffer computeCommandEncoderWithDescriptor: encoder_descriptor]; + encoder = [command_buffer computeCommandEncoderWithDescriptor: edesc]; } [encoder setComputePipelineState:ctx->pipeline_gelu]; @@ -682,7 +681,7 @@ void ggml_metal_graph_compute( case GGML_OP_SOFT_MAX: { if (encoder == nil) { - encoder = [command_buffer computeCommandEncoderWithDescriptor: encoder_descriptor]; + encoder = [command_buffer computeCommandEncoderWithDescriptor: edesc]; } const int nth = 32; @@ -700,7 +699,7 @@ void ggml_metal_graph_compute( case GGML_OP_DIAG_MASK_INF: { if (encoder == nil) { - encoder = [command_buffer computeCommandEncoderWithDescriptor: encoder_descriptor]; + encoder = [command_buffer computeCommandEncoderWithDescriptor: edesc]; } const int n_past = ((int32_t *)(dst->op_params))[0]; @@ -763,7 +762,7 @@ void ggml_metal_graph_compute( } } else { if (encoder == nil) { - encoder = [command_buffer computeCommandEncoderWithDescriptor: encoder_descriptor]; + encoder = [command_buffer computeCommandEncoderWithDescriptor: edesc]; } int nth0 = 32; @@ -890,7 +889,7 @@ void ggml_metal_graph_compute( case GGML_OP_GET_ROWS: { if (encoder == nil) { - encoder = [command_buffer computeCommandEncoderWithDescriptor: encoder_descriptor]; + encoder = [command_buffer computeCommandEncoderWithDescriptor: edesc]; } switch (src0->type) { @@ -919,7 +918,7 @@ void ggml_metal_graph_compute( case GGML_OP_RMS_NORM: { if (encoder == nil) { - encoder = [command_buffer computeCommandEncoderWithDescriptor: encoder_descriptor]; + encoder = [command_buffer computeCommandEncoderWithDescriptor: edesc]; } float eps; @@ -942,7 +941,7 @@ void ggml_metal_graph_compute( case GGML_OP_NORM: { if (encoder == nil) { - encoder = [command_buffer computeCommandEncoderWithDescriptor: encoder_descriptor]; + encoder = [command_buffer computeCommandEncoderWithDescriptor: edesc]; } const float eps = 1e-5f; @@ -964,7 +963,7 @@ void ggml_metal_graph_compute( case GGML_OP_ALIBI: { if (encoder == nil) { - encoder = [command_buffer computeCommandEncoderWithDescriptor: encoder_descriptor]; + encoder = [command_buffer computeCommandEncoderWithDescriptor: edesc]; } GGML_ASSERT((src0t == GGML_TYPE_F32)); @@ -1007,7 +1006,7 @@ void ggml_metal_graph_compute( case GGML_OP_ROPE: { if (encoder == nil) { - encoder = [command_buffer computeCommandEncoderWithDescriptor: encoder_descriptor]; + encoder = [command_buffer computeCommandEncoderWithDescriptor: edesc]; } const int n_past = ((int32_t *) dst->op_params)[0]; @@ -1051,7 +1050,7 @@ void ggml_metal_graph_compute( case GGML_OP_CONT: { if (encoder == nil) { - encoder = [command_buffer computeCommandEncoderWithDescriptor: encoder_descriptor]; + encoder = [command_buffer computeCommandEncoderWithDescriptor: edesc]; } const int nth = 32;