Skip to content

Commit ad3c3ad

Browse files
fmbenhassinephilwebb
fmbenhassine
authored and
philwebb
committed
Fix Spring Batch job restart parameters handling
Fix the `JobLauncherCommandLineRunner` to correctly deal with job parameters when restarting a job. Prior to this commit, we were was calling the `getNextJobParameters` method of the `JobParametersBuilder` from batch. This method was getting the previous parameters of the wrong job instance in a restart scenario. This commit fixes the issue by first getting the right job instance with the provided parameters, then restarting it. Closes gh-14933
1 parent d1ce315 commit ad3c3ad

File tree

3 files changed

+167
-9
lines changed

3 files changed

+167
-9
lines changed

‎spring-boot-project/spring-boot-autoconfigure/src/main/java/org/springframework/boot/autoconfigure/batch/BatchAutoConfiguration.java

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,7 @@
5656
* @author Dave Syer
5757
* @author Eddú Meléndez
5858
* @author Kazuki Shimizu
59+
* @author Mahmoud Ben Hassine
5960
*/
6061
@Configuration
6162
@ConditionalOnClass({ JobLauncher.class, DataSource.class, JdbcOperations.class })
@@ -88,9 +89,10 @@ public BatchDataSourceInitializer batchDataSourceInitializer(DataSource dataSour
8889
@ConditionalOnMissingBean
8990
@ConditionalOnProperty(prefix = "spring.batch.job", name = "enabled", havingValue = "true", matchIfMissing = true)
9091
public JobLauncherCommandLineRunner jobLauncherCommandLineRunner(
91-
JobLauncher jobLauncher, JobExplorer jobExplorer) {
92+
JobLauncher jobLauncher, JobExplorer jobExplorer,
93+
JobRepository jobRepository) {
9294
JobLauncherCommandLineRunner runner = new JobLauncherCommandLineRunner(
93-
jobLauncher, jobExplorer);
95+
jobLauncher, jobExplorer, jobRepository);
9496
String jobNames = this.properties.getJob().getNames();
9597
if (StringUtils.hasText(jobNames)) {
9698
runner.setJobNames(jobNames);

‎spring-boot-project/spring-boot-autoconfigure/src/main/java/org/springframework/boot/autoconfigure/batch/JobLauncherCommandLineRunner.java

Lines changed: 91 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright 2012-2017 the original author or authors.
2+
* Copyright 2012-2018 the original author or authors.
33
*
44
* Licensed under the Apache License, Version 2.0 (the "License");
55
* you may not use this file except in compliance with the License.
@@ -19,16 +19,21 @@
1919
import java.util.Arrays;
2020
import java.util.Collection;
2121
import java.util.Collections;
22+
import java.util.HashMap;
23+
import java.util.Map;
2224
import java.util.Properties;
2325

2426
import org.apache.commons.logging.Log;
2527
import org.apache.commons.logging.LogFactory;
2628

29+
import org.springframework.batch.core.BatchStatus;
2730
import org.springframework.batch.core.Job;
2831
import org.springframework.batch.core.JobExecution;
2932
import org.springframework.batch.core.JobExecutionException;
33+
import org.springframework.batch.core.JobParameter;
3034
import org.springframework.batch.core.JobParameters;
3135
import org.springframework.batch.core.JobParametersBuilder;
36+
import org.springframework.batch.core.JobParametersIncrementer;
3237
import org.springframework.batch.core.JobParametersInvalidException;
3338
import org.springframework.batch.core.configuration.JobRegistry;
3439
import org.springframework.batch.core.converter.DefaultJobParametersConverter;
@@ -39,6 +44,7 @@
3944
import org.springframework.batch.core.launch.NoSuchJobException;
4045
import org.springframework.batch.core.repository.JobExecutionAlreadyRunningException;
4146
import org.springframework.batch.core.repository.JobInstanceAlreadyCompleteException;
47+
import org.springframework.batch.core.repository.JobRepository;
4248
import org.springframework.batch.core.repository.JobRestartException;
4349
import org.springframework.beans.factory.annotation.Autowired;
4450
import org.springframework.boot.CommandLineRunner;
@@ -55,6 +61,7 @@
5561
*
5662
* @author Dave Syer
5763
* @author Jean-Pierre Bergamin
64+
* @author Mahmoud Ben Hassine
5865
*/
5966
public class JobLauncherCommandLineRunner
6067
implements CommandLineRunner, Ordered, ApplicationEventPublisherAware {
@@ -75,6 +82,8 @@ public class JobLauncherCommandLineRunner
7582

7683
private JobExplorer jobExplorer;
7784

85+
private JobRepository jobRepository;
86+
7887
private String jobNames;
7988

8089
private Collection<Job> jobs = Collections.emptySet();
@@ -83,12 +92,37 @@ public class JobLauncherCommandLineRunner
8392

8493
private ApplicationEventPublisher publisher;
8594

95+
/**
96+
* Create a new {@link JobLauncherCommandLineRunner}.
97+
* @param jobLauncher to launch jobs
98+
* @param jobExplorer to check the job repository for previous executions
99+
* @deprecated This constructor is deprecated in favor of
100+
* {@link JobLauncherCommandLineRunner#JobLauncherCommandLineRunner(JobLauncher, JobExplorer, JobRepository)}.
101+
* A job repository is required to check if a job instance exists with the given
102+
* parameters when running a job (which is not possible with the job explorer). This
103+
* constructor will be removed in a future version.
104+
*/
105+
@Deprecated
86106
public JobLauncherCommandLineRunner(JobLauncher jobLauncher,
87107
JobExplorer jobExplorer) {
88108
this.jobLauncher = jobLauncher;
89109
this.jobExplorer = jobExplorer;
90110
}
91111

112+
/**
113+
* Create a new {@link JobLauncherCommandLineRunner}.
114+
* @param jobLauncher to launch jobs
115+
* @param jobExplorer to check the job repository for previous executions
116+
* @param jobRepository to check if a job instance exists with the given parameters
117+
* when running a job
118+
*/
119+
public JobLauncherCommandLineRunner(JobLauncher jobLauncher, JobExplorer jobExplorer,
120+
JobRepository jobRepository) {
121+
this.jobLauncher = jobLauncher;
122+
this.jobExplorer = jobExplorer;
123+
this.jobRepository = jobRepository;
124+
}
125+
92126
public void setOrder(int order) {
93127
this.order = order;
94128
}
@@ -158,9 +192,39 @@ protected void execute(Job job, JobParameters jobParameters)
158192
throws JobExecutionAlreadyRunningException, JobRestartException,
159193
JobInstanceAlreadyCompleteException, JobParametersInvalidException,
160194
JobParametersNotFoundException {
161-
JobParameters nextParameters = new JobParametersBuilder(jobParameters,
162-
this.jobExplorer).getNextJobParameters(job).toJobParameters();
163-
JobExecution execution = this.jobLauncher.run(job, nextParameters);
195+
String jobName = job.getName();
196+
JobParameters parameters = jobParameters;
197+
boolean jobInstanceExists = this.jobRepository.isJobInstanceExists(jobName,
198+
parameters);
199+
if (jobInstanceExists) {
200+
JobExecution lastJobExecution = this.jobRepository
201+
.getLastJobExecution(jobName, jobParameters);
202+
if (lastJobExecution != null && isStoppedOrFailed(lastJobExecution)
203+
&& job.isRestartable()) {
204+
// Retry a failed or stopped execution with previous parameters
205+
JobParameters previousParameters = lastJobExecution.getJobParameters();
206+
/*
207+
* remove Non-identifying parameters from the previous execution's
208+
* parameters since there is no way to remove them programmatically. If
209+
* they are required (or need to be modified) on a restart, they need to
210+
* be (re)specified.
211+
*/
212+
JobParameters previousIdentifyingParameters = removeNonIdentifying(
213+
previousParameters);
214+
// merge additional parameters with previous ones (overriding those with
215+
// the same key)
216+
parameters = merge(previousIdentifyingParameters, jobParameters);
217+
}
218+
}
219+
else {
220+
JobParametersIncrementer incrementer = job.getJobParametersIncrementer();
221+
if (incrementer != null) {
222+
JobParameters nextParameters = new JobParametersBuilder(jobParameters,
223+
this.jobExplorer).getNextJobParameters(job).toJobParameters();
224+
parameters = merge(nextParameters, jobParameters);
225+
}
226+
}
227+
JobExecution execution = this.jobLauncher.run(job, parameters);
164228
if (this.publisher != null) {
165229
this.publisher.publishEvent(new JobExecutionEvent(execution));
166230
}
@@ -180,4 +244,27 @@ private void executeLocalJobs(JobParameters jobParameters)
180244
}
181245
}
182246

247+
private JobParameters removeNonIdentifying(JobParameters parameters) {
248+
Map<String, JobParameter> parameterMap = parameters.getParameters();
249+
HashMap<String, JobParameter> copy = new HashMap<>(parameterMap);
250+
for (Map.Entry<String, JobParameter> parameter : copy.entrySet()) {
251+
if (!parameter.getValue().isIdentifying()) {
252+
parameterMap.remove(parameter.getKey());
253+
}
254+
}
255+
return new JobParameters(parameterMap);
256+
}
257+
258+
private boolean isStoppedOrFailed(JobExecution execution) {
259+
BatchStatus status = execution.getStatus();
260+
return (status == BatchStatus.STOPPED || status == BatchStatus.FAILED);
261+
}
262+
263+
private JobParameters merge(JobParameters parameters, JobParameters additionals) {
264+
Map<String, JobParameter> merged = new HashMap<>();
265+
merged.putAll(parameters.getParameters());
266+
merged.putAll(additionals.getParameters());
267+
return new JobParameters(merged);
268+
}
269+
183270
}

‎spring-boot-project/spring-boot-autoconfigure/src/test/java/org/springframework/boot/autoconfigure/batch/JobLauncherCommandLineRunnerTests.java

Lines changed: 72 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright 2012-2017 the original author or authors.
2+
* Copyright 2012-2018 the original author or authors.
33
*
44
* Licensed under the Apache License, Version 2.0 (the "License");
55
* you may not use this file except in compliance with the License.
@@ -21,6 +21,8 @@
2121
import org.junit.Test;
2222

2323
import org.springframework.batch.core.Job;
24+
import org.springframework.batch.core.JobExecution;
25+
import org.springframework.batch.core.JobInstance;
2426
import org.springframework.batch.core.JobParameters;
2527
import org.springframework.batch.core.JobParametersBuilder;
2628
import org.springframework.batch.core.Step;
@@ -34,6 +36,7 @@
3436
import org.springframework.batch.core.launch.support.RunIdIncrementer;
3537
import org.springframework.batch.core.launch.support.SimpleJobLauncher;
3638
import org.springframework.batch.core.repository.JobRepository;
39+
import org.springframework.batch.core.repository.JobRestartException;
3740
import org.springframework.batch.core.repository.support.MapJobRepositoryFactoryBean;
3841
import org.springframework.batch.core.step.tasklet.Tasklet;
3942
import org.springframework.batch.support.transaction.ResourcelessTransactionManager;
@@ -43,12 +46,14 @@
4346
import org.springframework.transaction.PlatformTransactionManager;
4447

4548
import static org.assertj.core.api.Assertions.assertThat;
49+
import static org.assertj.core.api.Assertions.fail;
4650

4751
/**
4852
* Tests for {@link JobLauncherCommandLineRunner}.
4953
*
5054
* @author Dave Syer
5155
* @author Jean-Pierre Bergamin
56+
* @author Mahmoud Ben Hassine
5257
*/
5358
public class JobLauncherCommandLineRunnerTests {
5459

@@ -80,7 +85,8 @@ public void init() {
8085
this.step = this.steps.get("step").tasklet(tasklet).build();
8186
this.job = this.jobs.get("job").start(this.step).build();
8287
this.jobExplorer = this.context.getBean(JobExplorer.class);
83-
this.runner = new JobLauncherCommandLineRunner(jobLauncher, this.jobExplorer);
88+
this.runner = new JobLauncherCommandLineRunner(jobLauncher, this.jobExplorer,
89+
jobRepository);
8490
this.context.getBean(BatchConfiguration.class).clear();
8591
}
8692

@@ -113,10 +119,27 @@ public void retryFailedExecution() throws Exception {
113119
.start(this.steps.get("step").tasklet(throwingTasklet()).build())
114120
.incrementer(new RunIdIncrementer()).build();
115121
this.runner.execute(this.job, new JobParameters());
116-
this.runner.execute(this.job, new JobParameters());
122+
this.runner.execute(this.job,
123+
new JobParametersBuilder().addLong("run.id", 1L).toJobParameters());
117124
assertThat(this.jobExplorer.getJobInstances("job", 0, 100)).hasSize(1);
118125
}
119126

127+
@Test
128+
public void runDifferentInstances() throws Exception {
129+
this.job = this.jobs.get("job")
130+
.start(this.steps.get("step").tasklet(throwingTasklet()).build()).build();
131+
// start a job instance
132+
JobParameters jobParameters = new JobParametersBuilder().addString("name", "foo")
133+
.toJobParameters();
134+
this.runner.execute(this.job, jobParameters);
135+
assertThat(this.jobExplorer.getJobInstances("job", 0, 100)).hasSize(1);
136+
// start a different job instance
137+
JobParameters otherJobParameters = new JobParametersBuilder()
138+
.addString("name", "bar").toJobParameters();
139+
this.runner.execute(this.job, otherJobParameters);
140+
assertThat(this.jobExplorer.getJobInstances("job", 0, 100)).hasSize(2);
141+
}
142+
120143
@Test
121144
public void retryFailedExecutionOnNonRestartableJob() throws Exception {
122145
this.job = this.jobs.get("job").preventRestart()
@@ -127,6 +150,17 @@ public void retryFailedExecutionOnNonRestartableJob() throws Exception {
127150
// A failed job that is not restartable does not re-use the job params of
128151
// the last execution, but creates a new job instance when running it again.
129152
assertThat(this.jobExplorer.getJobInstances("job", 0, 100)).hasSize(2);
153+
try {
154+
// try to re-run a failed execution
155+
this.runner.execute(this.job,
156+
new JobParametersBuilder().addLong("run.id", 1L).toJobParameters());
157+
fail("expected JobRestartException");
158+
}
159+
catch (JobRestartException ex) {
160+
assertThat(ex.getMessage())
161+
.isEqualTo("JobInstance already exists and is not restartable");
162+
// expected
163+
}
130164
}
131165

132166
@Test
@@ -137,8 +171,43 @@ public void retryFailedExecutionWithNonIdentifyingParameters() throws Exception
137171
JobParameters jobParameters = new JobParametersBuilder().addLong("id", 1L, false)
138172
.addLong("foo", 2L, false).toJobParameters();
139173
this.runner.execute(this.job, jobParameters);
174+
assertThat(this.jobExplorer.getJobInstances("job", 0, 100)).hasSize(1);
175+
// try to re-run a failed execution with non identifying parameters
176+
this.runner.execute(this.job, new JobParametersBuilder(jobParameters)
177+
.addLong("run.id", 1L).toJobParameters());
178+
assertThat(this.jobExplorer.getJobInstances("job", 0, 100)).hasSize(1);
179+
}
180+
181+
@Test
182+
public void retryFailedExecutionWithDifferentNonIdentifyingParametersFromPreviousExecution()
183+
throws Exception {
184+
this.job = this.jobs.get("job")
185+
.start(this.steps.get("step").tasklet(throwingTasklet()).build())
186+
.incrementer(new RunIdIncrementer()).build();
187+
JobParameters jobParameters = new JobParametersBuilder().addLong("id", 1L, false)
188+
.addLong("foo", 2L, false).toJobParameters();
140189
this.runner.execute(this.job, jobParameters);
141190
assertThat(this.jobExplorer.getJobInstances("job", 0, 100)).hasSize(1);
191+
// try to re-run a failed execution with non identifying parameters
192+
this.runner.execute(this.job, new JobParametersBuilder().addLong("run.id", 1L)
193+
.addLong("id", 2L, false).addLong("foo", 3L, false).toJobParameters());
194+
assertThat(this.jobExplorer.getJobInstances("job", 0, 100)).hasSize(1);
195+
JobInstance jobInstance = this.jobExplorer.getJobInstance(0L);
196+
assertThat(this.jobExplorer.getJobExecutions(jobInstance)).hasSize(2);
197+
// first execution
198+
JobExecution firstJobExecution = this.jobExplorer.getJobExecution(0L);
199+
JobParameters parameters = firstJobExecution.getJobParameters();
200+
assertThat(parameters.getLong("run.id")).isEqualTo(1L);
201+
assertThat(parameters.getLong("id")).isEqualTo(1L);
202+
assertThat(parameters.getLong("foo")).isEqualTo(2L);
203+
// second execution
204+
JobExecution secondJobExecution = this.jobExplorer.getJobExecution(1L);
205+
parameters = secondJobExecution.getJobParameters();
206+
// identifying parameters should be the same as previous execution
207+
assertThat(parameters.getLong("run.id")).isEqualTo(1L);
208+
// non-identifying parameters should be the newly specified ones
209+
assertThat(parameters.getLong("id")).isEqualTo(2L);
210+
assertThat(parameters.getLong("foo")).isEqualTo(3L);
142211
}
143212

144213
private Tasklet throwingTasklet() {

0 commit comments

Comments
 (0)