SHDP-316 Remove ConfigurationUtils dependency on JobConf class

Author: Thomas Risberg

spring-hadoop-build-tests/src/test/java/org/springframework/data/hadoop/configuration/ConfigurationUtilsTest.java

@@ -0,0 +1,77 @@
+/*
+ * Copyright 2011-2013 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.springframework.data.hadoop.configuration;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapreduce.Job;
+import org.junit.Before;
+import org.junit.Test;
+import org.springframework.data.hadoop.util.VersionUtils;
+import org.springframework.util.ReflectionUtils;
+
+import java.lang.reflect.Method;
+import java.util.Properties;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * @author Thomas Risberg
+ */
+public class ConfigurationUtilsTest {
+
+	@Test
+	public void testConfigurationCreation() throws Exception {
+		Properties prop = new Properties();
+		prop.setProperty("name", "test1");
+		Configuration cfg = ConfigurationUtils.createFrom(null, prop);
+		assertEquals("test1", cfg.get("name"));
+		Configuration cfg2 = ConfigurationUtils.createFrom(cfg, prop);
+		assertEquals("test1", cfg2.get("name"));
+		assertTrue(cfg2 instanceof Configuration);
+		assertTrue(!(cfg2 instanceof JobConf));
+	}
+
+	@Test
+	public void testJobConfCreation() throws Exception {
+		Properties prop = new Properties();
+		prop.setProperty("name", "test2");
+		Configuration cfg = JobConfUtils.createFrom(null, prop);
+		assertEquals("test2", cfg.get("name"));
+		Configuration cfg2 = ConfigurationUtils.createFrom(cfg, prop);
+		assertEquals("test2", cfg2.get("name"));
+		assertTrue(cfg2 instanceof JobConf);
+	}
+
+	@Test
+	public void testConfigurationProperties() throws Exception {
+		Properties prop = new Properties();
+		prop.setProperty("mapred.reduce.tasks", "8");
+		prop.setProperty("mapred.map.tasks", "4");
+
+		Configuration cfg = ConfigurationUtils.createFrom(null, prop);
+
+		assertEquals("8", cfg.get("mapred.reduce.tasks"));
+		assertEquals("4", cfg.get("mapred.map.tasks"));
+		@SuppressWarnings("deprecation")
+		Job j = new Job(cfg);
+		assertEquals("8", j.getConfiguration().get("mapred.reduce.tasks"));
+		assertEquals("4", j.getConfiguration().get("mapred.map.tasks"));
+	}
+}

spring-hadoop-build-tests/src/test/java/org/springframework/data/hadoop/configuration/UrlInfiniteLoopTest.java

@@ -68,19 +68,4 @@ public void testInfiniteLoop() throws Exception {
 		assertNotNull(fs);
 	}
 
-	@Test
-	public void testConfigurationProperties() throws Exception {
-		Properties prop = new Properties();
-		prop.setProperty("mapred.reduce.tasks", "8");
-		prop.setProperty("mapred.map.tasks", "4");
-
-		Configuration cfg = ConfigurationUtils.createFrom(null, prop);
-
-		assertEquals("8", cfg.get("mapred.reduce.tasks"));
-		assertEquals("4", cfg.get("mapred.map.tasks"));
-		@SuppressWarnings("deprecation")
-		Job j = new Job(cfg);
-		assertEquals("8", j.getConfiguration().get("mapred.reduce.tasks"));
-		assertEquals("4", j.getConfiguration().get("mapred.map.tasks"));
-	}
 }

spring-hadoop-core/src/main/java/org/springframework/data/hadoop/configuration/ConfigurationUtils.java

@@ -23,7 +23,6 @@
 import java.util.Properties;
 
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.mapred.JobConf;
 import org.apache.hadoop.util.GenericOptionsParser;
 import org.springframework.core.io.Resource;
 import org.springframework.util.Assert;
@@ -63,28 +62,20 @@ public static void addProperties(Configuration configuration, Properties propert
 	public static Configuration createFrom(Configuration original, Properties properties) {
 		Configuration cfg = null;
 		if (original != null) {
-			cfg = (original instanceof JobConf ? new JobConf(original) : new Configuration(original));
+			if ("org.apache.hadoop.mapred.JobConf".equals(original.getClass().getName())) {
+				cfg = JobConfUtils.createFrom(original, properties);
+			} else {
+				cfg = new Configuration(original);
+				addProperties(cfg, properties);
+			}
 		}
 		else {
-			cfg = new JobConf();
+			cfg = new Configuration();
+			addProperties(cfg, properties);
 		}
-		addProperties(cfg, properties);
 		return cfg;
 	}
 
-	/**
-	 * Creates a new {@link JobConf} based on the given arguments. Identical to 
-	 * {@link #createFrom(Configuration, Properties)} but forces the use of
-	 * {@link JobConf}.
-	 * 
-	 * @param original initial configuration to read from. May be null. 
-	 * @param properties properties object to add to the newly created configuration. May be null.
-	 * @return newly created configuration based on the input parameters.
-	 */
-	public static JobConf createFrom(JobConf original, Properties properties) {
-		return (JobConf) createFrom((Configuration) original, properties);
-	}
-
 	/**
 	 * Returns a static {@link Properties} copy of the given configuration.
 	 * 
@@ -113,12 +104,12 @@ public static Properties asProperties(Configuration configuration) {
 	public static Configuration merge(Configuration one, Configuration two) {
 		if (one == null) {
 			if (two == null) {
-				return new JobConf();
+				return new Configuration();
 			}
-			return new JobConf(two);
+			return new Configuration(two);
 		}
 
-		Configuration c = new JobConf(one);
+		Configuration c = new Configuration(one);
 
 		if (two == null) {
 			return c;

spring-hadoop-core/src/main/java/org/springframework/data/hadoop/configuration/JobConfUtils.java

@@ -0,0 +1,79 @@
+/*
+ * Copyright 2011-2014 the original author or authors.
+ * 
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.springframework.data.hadoop.configuration;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.mapred.JobConf;
+
+import java.util.Map;
+import java.util.Properties;
+
+/**
+ * Reusable utility class for common {@link org.apache.hadoop.mapred.JobConf} operations.
+ *
+ * @author Thomas Risberg
+ */
+public abstract class JobConfUtils {
+
+	/**
+	 * Creates a new {@link org.apache.hadoop.mapred.JobConf} based on the given arguments.
+	 *
+	 * @param original initial configuration to read from. May be null.
+	 * @param properties properties object to add to the newly created configuration. May be null.
+	 * @return newly created configuration based on the input parameters.
+	 */
+	public static JobConf createFrom(Configuration original, Properties properties) {
+		JobConf cfg = null;
+		if (original != null) {
+			cfg = new JobConf(original);
+		}
+		else {
+			cfg = new JobConf();
+		}
+		ConfigurationUtils.addProperties(cfg, properties);
+		return cfg;
+	}
+
+	/**
+	 * Creates a new {@link org.apache.hadoop.conf.Configuration} by merging the given configurations.
+	 * Ordering is important - the second configuration overriding values in the first.
+	 * 
+	 * @param one configuration to read from. May be null.
+	 * @param two configuration to read from. May be null.
+	 * @return the result of merging the two configurations.
+	 */
+	public static JobConf merge(Configuration one, Configuration two) {
+		if (one == null) {
+			if (two == null) {
+				return new JobConf();
+			}
+			return new JobConf(two);
+		}
+
+		JobConf c = new JobConf(one);
+
+		if (two == null) {
+			return c;
+		}
+
+		for (Map.Entry<String, String> entry : two) {
+			c.set(entry.getKey(), entry.getValue());
+		}
+
+		return c;
+	}
+
+}

spring-hadoop-core/src/main/java/org/springframework/data/hadoop/mapreduce/HadoopCodeExecutor.java

@@ -25,7 +25,7 @@
 import org.springframework.beans.factory.BeanClassLoaderAware;
 import org.springframework.beans.factory.InitializingBean;
 import org.springframework.core.io.Resource;
-import org.springframework.data.hadoop.configuration.ConfigurationUtils;
+import org.springframework.data.hadoop.configuration.JobConfUtils;
 import org.springframework.data.hadoop.mapreduce.ExecutionUtils.ExitTrapped;
 import org.springframework.util.Assert;
 import org.springframework.util.ClassUtils;
@@ -116,7 +116,7 @@ public Integer run() throws Exception {
 
 
 	protected Configuration resolveConfiguration() throws Exception {
-		Configuration cfg = ConfigurationUtils.createFrom(configuration, properties);
+		Configuration cfg = JobConfUtils.createFrom(configuration, properties);
 		// add the jar if present
 		if (jar != null) {
 			String jarUrl = jar.getURL().toString();

spring-hadoop-core/src/main/java/org/springframework/data/hadoop/mapreduce/JobFactoryBean.java

@@ -43,7 +43,7 @@
 import org.springframework.beans.factory.InitializingBean;
 import org.springframework.core.io.Resource;
 import org.springframework.core.io.support.ResourcePatternResolver;
-import org.springframework.data.hadoop.configuration.ConfigurationUtils;
+import org.springframework.data.hadoop.configuration.JobConfUtils;
 import org.springframework.util.Assert;
 import org.springframework.util.ClassUtils;
 import org.springframework.util.CollectionUtils;
@@ -118,7 +118,7 @@ public boolean isSingleton() {
 
 	@SuppressWarnings({ "rawtypes", "deprecation" })
 	public void afterPropertiesSet() throws Exception {
-		final Configuration cfg = ConfigurationUtils.createFrom(configuration, properties);
+		final Configuration cfg = JobConfUtils.createFrom(configuration, properties);
 
 		buildGenericOptions(cfg);
 

spring-hadoop-core/src/main/java/org/springframework/data/hadoop/mapreduce/JobUtils.java

@@ -26,7 +26,7 @@
 import org.apache.hadoop.mapreduce.Job;
 import org.apache.hadoop.mapreduce.Job.JobState;
 import org.apache.hadoop.mapreduce.JobID;
-import org.springframework.data.hadoop.configuration.ConfigurationUtils;
+import org.springframework.data.hadoop.configuration.JobConfUtils;
 import org.springframework.data.hadoop.util.VersionUtils;
 import org.springframework.util.ReflectionUtils;
 
@@ -197,7 +197,7 @@ public static JobConf getJobConf(Job job) {
 			return (JobConf) configuration;
 		}
 
-		return (JobConf) ConfigurationUtils.createFrom(configuration, null);
+		return JobConfUtils.createFrom(configuration, null);
 	}
 
 	/**

spring-hadoop-core/src/main/java/org/springframework/data/hadoop/mapreduce/StreamJobFactoryBean.java

@@ -32,7 +32,7 @@
 import org.springframework.beans.factory.BeanNameAware;
 import org.springframework.beans.factory.FactoryBean;
 import org.springframework.beans.factory.InitializingBean;
-import org.springframework.data.hadoop.configuration.ConfigurationUtils;
+import org.springframework.data.hadoop.configuration.JobConfUtils;
 import org.springframework.util.Assert;
 import org.springframework.util.ObjectUtils;
 import org.springframework.util.ReflectionUtils;
@@ -80,7 +80,7 @@ public void afterPropertiesSet() throws Exception {
 		Assert.isTrue(!ObjectUtils.isEmpty(input), "at least one input required");
 		Assert.hasText(output, "the output is required");
 
-		final Configuration cfg = ConfigurationUtils.createFrom(configuration, properties);
+		final Configuration cfg = JobConfUtils.createFrom(configuration, properties);
 
 		buildGenericOptions(cfg);