Merge pull request #49 from uber/shadowing_sample

yux0 · web-flow · commit 9cdf420301e8 · 2021-06-11T11:20:34.000-07:00
Add shadowing samples
diff --git a/build.gradle b/build.gradle
@@ -35,7 +35,7 @@ repositories {
 dependencies {
     errorproneJavac("com.google.errorprone:javac:9+181-r4173-1")
     errorprone("com.google.errorprone:error_prone_core:2.3.1")
-    compile group: 'com.uber.cadence', name: 'cadence-client', version: '3.0.0'
+    compile group: 'com.uber.cadence', name: 'cadence-client', version: '3.1.0'
     compile group: 'commons-configuration', name: 'commons-configuration', version: '1.9'
     compile group: 'ch.qos.logback', name: 'logback-classic', version: '1.2.3'
     compile group: 'com.uber.m3', name: 'tally-core', version: '0.10.0'
diff --git a/src/main/java/com/uber/cadence/samples/hello/HelloAsync.java b/src/main/java/com/uber/cadence/samples/hello/HelloAsync.java
@@ -66,6 +66,7 @@ public static class GreetingWorkflowImpl implements GreetingWorkflow {
 
     @Override
     public String getGreeting(String name) {
+
       // Async.invoke takes method reference and activity parameters and returns Promise.
       Promise<String> hello = Async.function(activities::composeGreeting, "Hello", name);
       Promise<String> bye = Async.function(activities::composeGreeting, "Bye", name);
diff --git a/src/main/java/com/uber/cadence/samples/shadowing/README.md b/src/main/java/com/uber/cadence/samples/shadowing/README.md
@@ -0,0 +1,41 @@
+##Shadowing example:
+
+We provide Workflow Shadowing as a local testing solution. The example can be found in
+[HelloWorkflowShadowingTest](../../../../../../../test/java/com/uber/cadence/samples/hello/HelloWorkflowShadowingTest.java)
+
+
+This shadowing worker uses for detecting workflow non-deterministic error 
+prior to the workflow code deployment to prod. You can setup this shadowing worker in non-prod envrionment to shadow production traffic.
+
+More detail can be found: [design doc](https://github.com/uber/cadence/blob/master/docs/design/workflow-shadowing/2547-workflow-shadowing.md)
+
+1. To run this example, start a 0.21+ cadence server.
+
+2. Run a few HelloActivity workflow to generate workflow records.
+```
+./gradlew -q execute -PmainClass=com.uber.cadence.samples.hello.HelloActivity
+```
+
+3. Run the traffic shadowing
+```
+./gradlew -q execute -PmainClass=com.uber.cadence.samples.shadowing.ShadowTraffic
+```
+
+4. No non-deterministic error is expected in the stdout.
+
+5. Add a non backward compatible change to HelloActivity.
+```
+for example: add a timer between workflow start and activity schedule
+
+Workflow.sleep(1000);
+
+```
+
+6. Run the traffic shadowing
+```
+./gradlew -q execute -PmainClass=com.uber.cadence.samples.shadowing.ShadowTraffic
+```
+
+7. Non-deterministic error is expected in the stdout.
+
+
diff --git a/src/main/java/com/uber/cadence/samples/shadowing/ShadowTraffic.java b/src/main/java/com/uber/cadence/samples/shadowing/ShadowTraffic.java
@@ -0,0 +1,59 @@
+package com.uber.cadence.samples.shadowing;
+
+import com.google.common.collect.Lists;
+import com.uber.cadence.client.WorkflowClient;
+import com.uber.cadence.client.WorkflowClientOptions;
+import com.uber.cadence.samples.hello.HelloActivity;
+import com.uber.cadence.serviceclient.ClientOptions;
+import com.uber.cadence.serviceclient.WorkflowServiceTChannel;
+import com.uber.cadence.shadower.ExitCondition;
+import com.uber.cadence.shadower.Mode;
+import com.uber.cadence.worker.ShadowingOptions;
+import com.uber.cadence.worker.ShadowingWorker;
+import com.uber.cadence.worker.WorkerOptions;
+import com.uber.cadence.worker.WorkflowStatus;
+
+import java.util.concurrent.CountDownLatch;
+
+import static com.uber.cadence.samples.common.SampleConstants.DOMAIN;
+
+public class ShadowTraffic {
+    public static void main(String[] args) throws InterruptedException {
+        // Get a new client
+        // NOTE: to set a different options, you can do like this:
+        // ClientOptions.newBuilder().setRpcTimeout(5 * 1000).build();
+        WorkflowClient workflowClient =
+                WorkflowClient.newInstance(
+                        new WorkflowServiceTChannel(ClientOptions.defaultInstance()),
+                        WorkflowClientOptions.newBuilder().setDomain(DOMAIN).build());
+        ShadowingOptions options = ShadowingOptions
+                .newBuilder()
+                .setDomain(DOMAIN)
+                .setShadowMode(Mode.Normal)
+                .setWorkflowTypes(Lists.newArrayList("GreetingWorkflow::getGreeting"))
+                .setWorkflowStatuses(Lists.newArrayList(WorkflowStatus.OPEN, WorkflowStatus.CLOSED))
+                .setExitCondition(new ExitCondition().setExpirationIntervalInSeconds(60))
+                .build();
+
+        ShadowingWorker shadowingWorker = new ShadowingWorker(
+                workflowClient,
+                "HelloActivity",
+                WorkerOptions.defaultInstance(),
+                options);
+        shadowingWorker.registerWorkflowImplementationTypes(HelloActivity.GreetingWorkflowImpl.class);
+
+        CountDownLatch latch = new CountDownLatch(1);
+        // Execute a workflow waiting for it to complete.
+        Runnable runnable = () -> {
+            try {
+                shadowingWorker.start();
+            } catch (Exception e) {
+                System.out.println("Failed to start shadowing workflow");
+                System.out.println(e);
+                latch.countDown();
+            }
+        };
+        runnable.run();
+        latch.await();
+    }
+}
diff --git a/src/test/java/com/uber/cadence/samples/hello/HelloWorkflowShadowingTest.java b/src/test/java/com/uber/cadence/samples/hello/HelloWorkflowShadowingTest.java
@@ -0,0 +1,52 @@
+/*
+ *  Copyright 2012-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ *  Modifications copyright (C) 2017-2021 Uber Technologies, Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License"). You may not
+ *  use this file except in compliance with the License. A copy of the License is
+ *  located at
+ *
+ *  http://aws.amazon.com/apache2.0
+ *
+ *  or in the "license" file accompanying this file. This file is distributed on
+ *  an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ *  express or implied. See the License for the specific language governing
+ *  permissions and limitations under the License.
+ */
+
+package com.uber.cadence.samples.hello;
+
+import com.google.common.collect.Lists;
+import com.uber.cadence.serviceclient.ClientOptions;
+import com.uber.cadence.serviceclient.IWorkflowService;
+import com.uber.cadence.serviceclient.WorkflowServiceTChannel;
+import com.uber.cadence.shadower.ExitCondition;
+import com.uber.cadence.shadower.Mode;
+import com.uber.cadence.testing.WorkflowShadower;
+import com.uber.cadence.worker.ShadowingOptions;
+import com.uber.cadence.worker.WorkflowStatus;
+import org.junit.Test;
+
+import static com.uber.cadence.samples.common.SampleConstants.DOMAIN;
+import static com.uber.cadence.samples.hello.HelloActivity.TASK_LIST;
+
+public class HelloWorkflowShadowingTest {
+  @Test
+  public void testShadowing() throws Throwable {
+    IWorkflowService service = new WorkflowServiceTChannel(ClientOptions.defaultInstance());
+
+    ShadowingOptions options = ShadowingOptions
+            .newBuilder()
+            .setDomain(DOMAIN)
+            .setShadowMode(Mode.Normal)
+            .setWorkflowTypes(Lists.newArrayList("GreetingWorkflow::getGreeting"))
+            .setWorkflowStatuses(Lists.newArrayList(WorkflowStatus.OPEN, WorkflowStatus.CLOSED))
+            .setExitCondition(new ExitCondition().setExpirationIntervalInSeconds(60))
+            .build();
+    WorkflowShadower shadower = new WorkflowShadower(service, options, TASK_LIST);
+    shadower.registerWorkflowImplementationTypes(HelloActivity.GreetingWorkflowImpl.class);
+
+    shadower.run();
+  }
+}
diff --git a/src/test/resources/history.json b/src/test/resources/history.json
@@ -0,0 +1 @@
+[{"eventId":1,"timestamp":1615406829683961000,"eventType":"WorkflowExecutionStarted","version":-24,"taskId":1048576,"workflowExecutionStartedEventAttributes":{"workflowType":{"name":"FileProcessingWorkflow::processFile"},"taskList":{"name":"FileProcessing"},"input":"WyJodHRwOi8vd3d3Lmdvb2dsZS5jb20vIiwiaHR0cDovL2R1bW15Il0=","executionStartToCloseTimeoutSeconds":30,"taskStartToCloseTimeoutSeconds":10,"continuedExecutionRunId":"","originalExecutionRunId":"a3073e45-106d-4463-b1ae-1516070e0136","identity":"","firstExecutionRunId":"a3073e45-106d-4463-b1ae-1516070e0136","attempt":0,"cronSchedule":"","firstDecisionTaskBackoffSeconds":0}},{"eventId":2,"timestamp":1615406829683990000,"eventType":"DecisionTaskScheduled","version":-24,"taskId":1048577,"decisionTaskScheduledEventAttributes":{"taskList":{"name":"FileProcessing"},"startToCloseTimeoutSeconds":10,"attempt":0}},{"eventId":3,"timestamp":1615406829704269000,"eventType":"DecisionTaskStarted","version":-24,"taskId":1048582,"decisionTaskStartedEventAttributes":{"scheduledEventId":2,"identity":"14493@yx-C02ZD32HLVDQ","requestId":"4a5d6a33-d6f7-469e-a040-1d20004f0aaf"}},{"eventId":4,"timestamp":1615406829872782000,"eventType":"DecisionTaskCompleted","version":-24,"taskId":1048585,"decisionTaskCompletedEventAttributes":{"scheduledEventId":2,"startedEventId":3,"identity":"14493@yx-C02ZD32HLVDQ","binaryChecksum":""}},{"eventId":5,"timestamp":1615406829872867000,"eventType":"MarkerRecorded","version":-24,"taskId":1048586,"markerRecordedEventAttributes":{"markerName":"MutableSideEffect","details":"eyJpbml0aWFsSW50ZXJ2YWwiOnsic2Vjb25kcyI6MSwibmFub3MiOjB9LCJiYWNrb2ZmQ29lZmZpY2llbnQiOjIuMCwiZXhwaXJhdGlvbiI6eyJzZWNvbmRzIjoxMCwibmFub3MiOjB9LCJtYXhpbXVtQXR0ZW1wdHMiOjAsIm1heGltdW1JbnRlcnZhbCI6bnVsbCwiZG9Ob3RSZXRyeSI6bnVsbH0=","decisionTaskCompletedEventId":4,"header":{"fields":{"MutableMarkerHeader":"eyJpZCI6IjBmYjhhZmNjLWNmYTUtMzFhZi1iZDhiLTI1OGUwMTcxZTU2MyIsImV2ZW50SWQiOjUsImFjY2Vzc0NvdW50IjowfQ=="}}}},{"eventId":6,"timestamp":1615406829872878000,"eventType":"ActivityTaskScheduled","version":-24,"taskId":1048587,"activityTaskScheduledEventAttributes":{"activityId":"1","activityType":{"name":"StoreActivities::download"},"domain":"","taskList":{"name":"FileProcessing"},"input":"Imh0dHA6Ly93d3cuZ29vZ2xlLmNvbS8i","scheduleToCloseTimeoutSeconds":10,"scheduleToStartTimeoutSeconds":10,"startToCloseTimeoutSeconds":10,"heartbeatTimeoutSeconds":10,"decisionTaskCompletedEventId":4}},{"eventId":7,"timestamp":1615406829881959000,"eventType":"ActivityTaskStarted","version":-24,"taskId":1048591,"activityTaskStartedEventAttributes":{"scheduledEventId":6,"identity":"14493@yx-C02ZD32HLVDQ","requestId":"6a837403-a3c5-4576-931f-a4d181f9b7f1","attempt":0,"lastFailureReason":""}},{"eventId":8,"timestamp":1615406830028265000,"eventType":"ActivityTaskCompleted","version":-24,"taskId":1048594,"activityTaskCompletedEventAttributes":{"result":"eyJob3N0VGFza0xpc3QiOiIxNDQ5M0B5eC1DMDJaRDMySExWRFEiLCJmaWxlTmFtZSI6Ii92YXIvZm9sZGVycy9fbi8zYnFiemR2ZDMwZDFnNjJqcV8xeDh5MTQwMDAwZ24vVC8xNjE1NDA2ODMwMDE4LTAvZG93bmxvYWRlZCJ9","scheduledEventId":6,"startedEventId":7,"identity":"14493@yx-C02ZD32HLVDQ"}},{"eventId":9,"timestamp":1615406830028320000,"eventType":"DecisionTaskScheduled","version":-24,"taskId":1048596,"decisionTaskScheduledEventAttributes":{"taskList":{"name":"yx-C02ZD32HLVDQ:6a756b8f-9140-48de-8381-3fb0787127d2"},"startToCloseTimeoutSeconds":10,"attempt":0}},{"eventId":10,"timestamp":1615406830035061000,"eventType":"DecisionTaskStarted","version":-24,"taskId":1048600,"decisionTaskStartedEventAttributes":{"scheduledEventId":9,"identity":"6a756b8f-9140-48de-8381-3fb0787127d2","requestId":"9c3fc049-d99a-4b4c-8d78-8bb37260d410"}},{"eventId":11,"timestamp":1615406830057168000,"eventType":"DecisionTaskCompleted","version":-24,"taskId":1048603,"decisionTaskCompletedEventAttributes":{"scheduledEventId":9,"startedEventId":10,"identity":"14493@yx-C02ZD32HLVDQ","binaryChecksum":""}},{"eventId":12,"timestamp":1615406830057191000,"eventType":"ActivityTaskScheduled","version":-24,"taskId":1048604,"activityTaskScheduledEventAttributes":{"activityId":"2","activityType":{"name":"StoreActivities::process"},"domain":"","taskList":{"name":"14493@yx-C02ZD32HLVDQ"},"input":"Ii92YXIvZm9sZGVycy9fbi8zYnFiemR2ZDMwZDFnNjJqcV8xeDh5MTQwMDAwZ24vVC8xNjE1NDA2ODMwMDE4LTAvZG93bmxvYWRlZCI=","scheduleToCloseTimeoutSeconds":10,"scheduleToStartTimeoutSeconds":10,"startToCloseTimeoutSeconds":10,"heartbeatTimeoutSeconds":10,"decisionTaskCompletedEventId":11}},{"eventId":13,"timestamp":1615406830062863000,"eventType":"ActivityTaskStarted","version":-24,"taskId":1048608,"activityTaskStartedEventAttributes":{"scheduledEventId":12,"identity":"14493@yx-C02ZD32HLVDQ","requestId":"fc07913a-32dd-42dd-97f2-0c4bba882546","attempt":0,"lastFailureReason":""}},{"eventId":14,"timestamp":1615406830071599000,"eventType":"ActivityTaskCompleted","version":-24,"taskId":1048611,"activityTaskCompletedEventAttributes":{"result":"Ii92YXIvZm9sZGVycy9fbi8zYnFiemR2ZDMwZDFnNjJqcV8xeDh5MTQwMDAwZ24vVC8xNjE1NDA2ODMwMDE4LTAvcHJvY2Vzc2VkIg==","scheduledEventId":12,"startedEventId":13,"identity":"14493@yx-C02ZD32HLVDQ"}},{"eventId":15,"timestamp":1615406830071609000,"eventType":"DecisionTaskScheduled","version":-24,"taskId":1048613,"decisionTaskScheduledEventAttributes":{"taskList":{"name":"yx-C02ZD32HLVDQ:6a756b8f-9140-48de-8381-3fb0787127d2"},"startToCloseTimeoutSeconds":10,"attempt":0}},{"eventId":16,"timestamp":1615406830077115000,"eventType":"DecisionTaskStarted","version":-24,"taskId":1048617,"decisionTaskStartedEventAttributes":{"scheduledEventId":15,"identity":"6a756b8f-9140-48de-8381-3fb0787127d2","requestId":"8b60aa4a-beac-47ee-95ab-7f6a24139a96"}},{"eventId":17,"timestamp":1615406830085797000,"eventType":"DecisionTaskCompleted","version":-24,"taskId":1048620,"decisionTaskCompletedEventAttributes":{"scheduledEventId":15,"startedEventId":16,"identity":"14493@yx-C02ZD32HLVDQ","binaryChecksum":""}},{"eventId":18,"timestamp":1615406830085819000,"eventType":"ActivityTaskScheduled","version":-24,"taskId":1048621,"activityTaskScheduledEventAttributes":{"activityId":"3","activityType":{"name":"StoreActivities::upload"},"domain":"","taskList":{"name":"14493@yx-C02ZD32HLVDQ"},"input":"WyIvdmFyL2ZvbGRlcnMvX24vM2JxYnpkdmQzMGQxZzYyanFfMXg4eTE0MDAwMGduL1QvMTYxNTQwNjgzMDAxOC0wL3Byb2Nlc3NlZCIsImh0dHA6Ly9kdW1teSJd","scheduleToCloseTimeoutSeconds":10,"scheduleToStartTimeoutSeconds":10,"startToCloseTimeoutSeconds":10,"heartbeatTimeoutSeconds":10,"decisionTaskCompletedEventId":17}},{"eventId":19,"timestamp":1615406830091214000,"eventType":"ActivityTaskStarted","version":-24,"taskId":1048625,"activityTaskStartedEventAttributes":{"scheduledEventId":18,"identity":"14493@yx-C02ZD32HLVDQ","requestId":"5994a514-8fdc-4a20-9133-f32ad8b40800","attempt":0,"lastFailureReason":""}},{"eventId":20,"timestamp":1615406830097800000,"eventType":"ActivityTaskCompleted","version":-24,"taskId":1048628,"activityTaskCompletedEventAttributes":{"scheduledEventId":18,"startedEventId":19,"identity":"14493@yx-C02ZD32HLVDQ"}},{"eventId":21,"timestamp":1615406830097810000,"eventType":"DecisionTaskScheduled","version":-24,"taskId":1048630,"decisionTaskScheduledEventAttributes":{"taskList":{"name":"yx-C02ZD32HLVDQ:6a756b8f-9140-48de-8381-3fb0787127d2"},"startToCloseTimeoutSeconds":10,"attempt":0}},{"eventId":22,"timestamp":1615406830103719000,"eventType":"DecisionTaskStarted","version":-24,"taskId":1048634,"decisionTaskStartedEventAttributes":{"scheduledEventId":21,"identity":"6a756b8f-9140-48de-8381-3fb0787127d2","requestId":"d70fc76c-a921-4b68-83d9-f87ed1bbf9fb"}},{"eventId":23,"timestamp":1615406830115206000,"eventType":"DecisionTaskCompleted","version":-24,"taskId":1048637,"decisionTaskCompletedEventAttributes":{"scheduledEventId":21,"startedEventId":22,"identity":"14493@yx-C02ZD32HLVDQ","binaryChecksum":""}},{"eventId":24,"timestamp":1615406830115238000,"eventType":"WorkflowExecutionCompleted","version":-24,"taskId":1048638,"workflowExecutionCompletedEventAttributes":{"decisionTaskCompletedEventId":23}}]

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	+[{"eventId":1,"timestamp":1615406829683961000,"eventType":"WorkflowExecutionStarted","version":-24,"taskId":1048576,"workflowExecutionStartedEventAttributes":{"workflowType":{"name":"FileProcessingWorkflow::processFile"},"taskList":{"name":"FileProcessing"},"input":"WyJodHRwOi8vd3d3Lmdvb2dsZS5jb20vIiwiaHR0cDovL2R1bW15Il0=","executionStartToCloseTimeoutSeconds":30,"taskStartToCloseTimeoutSeconds":10,"continuedExecutionRunId":"","originalExecutionRunId":"a3073e45-106d-4463-b1ae-1516070e0136","identity":"","firstExecutionRunId":"a3073e45-106d-4463-b1ae-1516070e0136","attempt":0,"cronSchedule":"","firstDecisionTaskBackoffSeconds":0}},{"eventId":2,"timestamp":1615406829683990000,"eventType":"DecisionTaskScheduled","version":-24,"taskId":1048577,"decisionTaskScheduledEventAttributes":{"taskList":{"name":"FileProcessing"},"startToCloseTimeoutSeconds":10,"attempt":0}},{"eventId":3,"timestamp":1615406829704269000,"eventType":"DecisionTaskStarted","version":-24,"taskId":1048582,"decisionTaskStartedEventAttributes":{"scheduledEventId":2,"identity":"14493@yx-C02ZD32HLVDQ","requestId":"4a5d6a33-d6f7-469e-a040-1d20004f0aaf"}},{"eventId":4,"timestamp":1615406829872782000,"eventType":"DecisionTaskCompleted","version":-24,"taskId":1048585,"decisionTaskCompletedEventAttributes":{"scheduledEventId":2,"startedEventId":3,"identity":"14493@yx-C02ZD32HLVDQ","binaryChecksum":""}},{"eventId":5,"timestamp":1615406829872867000,"eventType":"MarkerRecorded","version":-24,"taskId":1048586,"markerRecordedEventAttributes":{"markerName":"MutableSideEffect","details":"eyJpbml0aWFsSW50ZXJ2YWwiOnsic2Vjb25kcyI6MSwibmFub3MiOjB9LCJiYWNrb2ZmQ29lZmZpY2llbnQiOjIuMCwiZXhwaXJhdGlvbiI6eyJzZWNvbmRzIjoxMCwibmFub3MiOjB9LCJtYXhpbXVtQXR0ZW1wdHMiOjAsIm1heGltdW1JbnRlcnZhbCI6bnVsbCwiZG9Ob3RSZXRyeSI6bnVsbH0=","decisionTaskCompletedEventId":4,"header":{"fields":{"MutableMarkerHeader":"eyJpZCI6IjBmYjhhZmNjLWNmYTUtMzFhZi1iZDhiLTI1OGUwMTcxZTU2MyIsImV2ZW50SWQiOjUsImFjY2Vzc0NvdW50IjowfQ=="}}}},{"eventId":6,"timestamp":1615406829872878000,"eventType":"ActivityTaskScheduled","version":-24,"taskId":1048587,"activityTaskScheduledEventAttributes":{"activityId":"1","activityType":{"name":"StoreActivities::download"},"domain":"","taskList":{"name":"FileProcessing"},"input":"Imh0dHA6Ly93d3cuZ29vZ2xlLmNvbS8i","scheduleToCloseTimeoutSeconds":10,"scheduleToStartTimeoutSeconds":10,"startToCloseTimeoutSeconds":10,"heartbeatTimeoutSeconds":10,"decisionTaskCompletedEventId":4}},{"eventId":7,"timestamp":1615406829881959000,"eventType":"ActivityTaskStarted","version":-24,"taskId":1048591,"activityTaskStartedEventAttributes":{"scheduledEventId":6,"identity":"14493@yx-C02ZD32HLVDQ","requestId":"6a837403-a3c5-4576-931f-a4d181f9b7f1","attempt":0,"lastFailureReason":""}},{"eventId":8,"timestamp":1615406830028265000,"eventType":"ActivityTaskCompleted","version":-24,"taskId":1048594,"activityTaskCompletedEventAttributes":{"result":"eyJob3N0VGFza0xpc3QiOiIxNDQ5M0B5eC1DMDJaRDMySExWRFEiLCJmaWxlTmFtZSI6Ii92YXIvZm9sZGVycy9fbi8zYnFiemR2ZDMwZDFnNjJqcV8xeDh5MTQwMDAwZ24vVC8xNjE1NDA2ODMwMDE4LTAvZG93bmxvYWRlZCJ9","scheduledEventId":6,"startedEventId":7,"identity":"14493@yx-C02ZD32HLVDQ"}},{"eventId":9,"timestamp":1615406830028320000,"eventType":"DecisionTaskScheduled","version":-24,"taskId":1048596,"decisionTaskScheduledEventAttributes":{"taskList":{"name":"yx-C02ZD32HLVDQ:6a756b8f-9140-48de-8381-3fb0787127d2"},"startToCloseTimeoutSeconds":10,"attempt":0}},{"eventId":10,"timestamp":1615406830035061000,"eventType":"DecisionTaskStarted","version":-24,"taskId":1048600,"decisionTaskStartedEventAttributes":{"scheduledEventId":9,"identity":"6a756b8f-9140-48de-8381-3fb0787127d2","requestId":"9c3fc049-d99a-4b4c-8d78-8bb37260d410"}},{"eventId":11,"timestamp":1615406830057168000,"eventType":"DecisionTaskCompleted","version":-24,"taskId":1048603,"decisionTaskCompletedEventAttributes":{"scheduledEventId":9,"startedEventId":10,"identity":"14493@yx-C02ZD32HLVDQ","binaryChecksum":""}},{"eventId":12,"timestamp":1615406830057191000,"eventType":"ActivityTaskScheduled","version":-24,"taskId":1048604,"activityTaskScheduledEventAttributes":{"activityId":"2","activityType":{"name":"StoreActivities::process"},"domain":"","taskList":{"name":"14493@yx-C02ZD32HLVDQ"},"input":"Ii92YXIvZm9sZGVycy9fbi8zYnFiemR2ZDMwZDFnNjJqcV8xeDh5MTQwMDAwZ24vVC8xNjE1NDA2ODMwMDE4LTAvZG93bmxvYWRlZCI=","scheduleToCloseTimeoutSeconds":10,"scheduleToStartTimeoutSeconds":10,"startToCloseTimeoutSeconds":10,"heartbeatTimeoutSeconds":10,"decisionTaskCompletedEventId":11}},{"eventId":13,"timestamp":1615406830062863000,"eventType":"ActivityTaskStarted","version":-24,"taskId":1048608,"activityTaskStartedEventAttributes":{"scheduledEventId":12,"identity":"14493@yx-C02ZD32HLVDQ","requestId":"fc07913a-32dd-42dd-97f2-0c4bba882546","attempt":0,"lastFailureReason":""}},{"eventId":14,"timestamp":1615406830071599000,"eventType":"ActivityTaskCompleted","version":-24,"taskId":1048611,"activityTaskCompletedEventAttributes":{"result":"Ii92YXIvZm9sZGVycy9fbi8zYnFiemR2ZDMwZDFnNjJqcV8xeDh5MTQwMDAwZ24vVC8xNjE1NDA2ODMwMDE4LTAvcHJvY2Vzc2VkIg==","scheduledEventId":12,"startedEventId":13,"identity":"14493@yx-C02ZD32HLVDQ"}},{"eventId":15,"timestamp":1615406830071609000,"eventType":"DecisionTaskScheduled","version":-24,"taskId":1048613,"decisionTaskScheduledEventAttributes":{"taskList":{"name":"yx-C02ZD32HLVDQ:6a756b8f-9140-48de-8381-3fb0787127d2"},"startToCloseTimeoutSeconds":10,"attempt":0}},{"eventId":16,"timestamp":1615406830077115000,"eventType":"DecisionTaskStarted","version":-24,"taskId":1048617,"decisionTaskStartedEventAttributes":{"scheduledEventId":15,"identity":"6a756b8f-9140-48de-8381-3fb0787127d2","requestId":"8b60aa4a-beac-47ee-95ab-7f6a24139a96"}},{"eventId":17,"timestamp":1615406830085797000,"eventType":"DecisionTaskCompleted","version":-24,"taskId":1048620,"decisionTaskCompletedEventAttributes":{"scheduledEventId":15,"startedEventId":16,"identity":"14493@yx-C02ZD32HLVDQ","binaryChecksum":""}},{"eventId":18,"timestamp":1615406830085819000,"eventType":"ActivityTaskScheduled","version":-24,"taskId":1048621,"activityTaskScheduledEventAttributes":{"activityId":"3","activityType":{"name":"StoreActivities::upload"},"domain":"","taskList":{"name":"14493@yx-C02ZD32HLVDQ"},"input":"WyIvdmFyL2ZvbGRlcnMvX24vM2JxYnpkdmQzMGQxZzYyanFfMXg4eTE0MDAwMGduL1QvMTYxNTQwNjgzMDAxOC0wL3Byb2Nlc3NlZCIsImh0dHA6Ly9kdW1teSJd","scheduleToCloseTimeoutSeconds":10,"scheduleToStartTimeoutSeconds":10,"startToCloseTimeoutSeconds":10,"heartbeatTimeoutSeconds":10,"decisionTaskCompletedEventId":17}},{"eventId":19,"timestamp":1615406830091214000,"eventType":"ActivityTaskStarted","version":-24,"taskId":1048625,"activityTaskStartedEventAttributes":{"scheduledEventId":18,"identity":"14493@yx-C02ZD32HLVDQ","requestId":"5994a514-8fdc-4a20-9133-f32ad8b40800","attempt":0,"lastFailureReason":""}},{"eventId":20,"timestamp":1615406830097800000,"eventType":"ActivityTaskCompleted","version":-24,"taskId":1048628,"activityTaskCompletedEventAttributes":{"scheduledEventId":18,"startedEventId":19,"identity":"14493@yx-C02ZD32HLVDQ"}},{"eventId":21,"timestamp":1615406830097810000,"eventType":"DecisionTaskScheduled","version":-24,"taskId":1048630,"decisionTaskScheduledEventAttributes":{"taskList":{"name":"yx-C02ZD32HLVDQ:6a756b8f-9140-48de-8381-3fb0787127d2"},"startToCloseTimeoutSeconds":10,"attempt":0}},{"eventId":22,"timestamp":1615406830103719000,"eventType":"DecisionTaskStarted","version":-24,"taskId":1048634,"decisionTaskStartedEventAttributes":{"scheduledEventId":21,"identity":"6a756b8f-9140-48de-8381-3fb0787127d2","requestId":"d70fc76c-a921-4b68-83d9-f87ed1bbf9fb"}},{"eventId":23,"timestamp":1615406830115206000,"eventType":"DecisionTaskCompleted","version":-24,"taskId":1048637,"decisionTaskCompletedEventAttributes":{"scheduledEventId":21,"startedEventId":22,"identity":"14493@yx-C02ZD32HLVDQ","binaryChecksum":""}},{"eventId":24,"timestamp":1615406830115238000,"eventType":"WorkflowExecutionCompleted","version":-24,"taskId":1048638,"workflowExecutionCompletedEventAttributes":{"decisionTaskCompletedEventId":23}}]