From 8071dbb9c6b4a654d5e1e7c8e3b4d2ca1a736d53 Mon Sep 17 00:00:00 2001 From: Viraj Jasani Date: Thu, 7 Oct 2021 07:28:29 +0530 Subject: [PATCH] HADOOP-17950. Provide replacement for deprecated APIs of commons-io IOUtils (#3515) Signed-off-by: Akira Ajisaka --- .../org/apache/hadoop/security/KDiag.java | 3 ++- .../apache/hadoop/security/ProviderUtils.java | 4 ++- .../hadoop/fs/shell/TestTextCommand.java | 3 ++- .../org/apache/hadoop/security/TestKDiag.java | 3 ++- .../web/TestWebDelegationToken.java | 27 ++++++++++++------- .../util/TestApplicationClassLoader.java | 3 ++- .../diskbalancer/command/CancelCommand.java | 3 ++- .../diskbalancer/command/ExecuteCommand.java | 4 +-- .../hadoop/fs/shell/TestHdfsTextCommand.java | 3 ++- .../TestOfflineImageViewerForXAttr.java | 13 ++++++--- .../apache/hadoop/hdfs/web/TestWebHDFS.java | 4 ++- .../hadoop/mapred/TestJavaSerialization.java | 4 ++- .../uploader/TestFrameworkUploader.java | 4 ++- .../plugin/resourcegz/ResourceGzMojo.java | 3 ++- .../hadoop/tools/TestHadoopArchiveLogs.java | 4 ++- .../hadoop/yarn/service/ServiceScheduler.java | 3 ++- .../yarn/service/provider/ProviderUtils.java | 4 ++- .../yarn/util/DockerClientConfigHandler.java | 2 +- 18 files changed, 64 insertions(+), 30 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/KDiag.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/KDiag.java index f759dbdb44..ddadfbf217 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/KDiag.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/KDiag.java @@ -46,6 +46,7 @@ import java.io.PrintWriter; import java.lang.reflect.InvocationTargetException; import java.net.InetAddress; +import java.nio.charset.Charset; import java.nio.file.Files; import java.security.NoSuchAlgorithmException; import java.util.ArrayList; @@ -923,7 +924,7 @@ private void printEnv(String variable) { */ private void dump(File file) throws IOException { try (InputStream in = Files.newInputStream(file.toPath())) { - for (String line : IOUtils.readLines(in)) { + for (String line : IOUtils.readLines(in, Charset.defaultCharset())) { println("%s", line); } } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ProviderUtils.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ProviderUtils.java index 23efffcd98..86e065e04d 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ProviderUtils.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ProviderUtils.java @@ -23,6 +23,7 @@ import java.net.URI; import java.net.URISyntaxException; import java.net.URL; +import java.nio.charset.Charset; import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.commons.io.IOUtils; @@ -224,7 +225,8 @@ public static char[] locatePassword(String envWithPass, String fileWithPass) throw new IOException("Password file does not exist"); } try (InputStream is = pwdFile.openStream()) { - pass = IOUtils.toString(is).trim().toCharArray(); + pass = IOUtils.toString(is, Charset.defaultCharset()).trim() + .toCharArray(); } } } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestTextCommand.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestTextCommand.java index 7b84836372..a9e7f57a29 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestTextCommand.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestTextCommand.java @@ -27,6 +27,7 @@ import java.io.StringWriter; import java.lang.reflect.Method; import java.net.URI; +import java.nio.charset.Charset; import java.nio.file.Files; import java.nio.file.Paths; @@ -124,7 +125,7 @@ public InputStream getInputStream(PathData item) throws IOException { private String inputStreamToString(InputStream stream) throws IOException { StringWriter writer = new StringWriter(); - IOUtils.copy(stream, writer); + IOUtils.copy(stream, writer, Charset.defaultCharset()); return writer.toString(); } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestKDiag.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestKDiag.java index 297d240d30..3a94bd99b8 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestKDiag.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestKDiag.java @@ -36,6 +36,7 @@ import java.io.File; import java.io.FileInputStream; import java.io.IOException; +import java.nio.charset.Charset; import java.util.Properties; import java.util.concurrent.TimeUnit; @@ -235,7 +236,7 @@ public void testKeytabUnknownPrincipal() throws Throwable { */ private void dump(File file) throws IOException { try (FileInputStream in = new FileInputStream(file)) { - for (String line : IOUtils.readLines(in)) { + for (String line : IOUtils.readLines(in, Charset.defaultCharset())) { LOG.info(line); } } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/web/TestWebDelegationToken.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/web/TestWebDelegationToken.java index 9b5bd22dbe..366c114fc1 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/web/TestWebDelegationToken.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/web/TestWebDelegationToken.java @@ -64,6 +64,7 @@ import java.io.Writer; import java.net.HttpURLConnection; import java.net.URL; +import java.nio.charset.Charset; import java.security.Principal; import java.security.PrivilegedActionException; import java.security.PrivilegedExceptionAction; @@ -554,7 +555,8 @@ public Void run() throws Exception { HttpURLConnection conn = aUrl.openConnection(url, token); Assert.assertEquals(HttpURLConnection.HTTP_OK, conn.getResponseCode()); - List ret = IOUtils.readLines(conn.getInputStream()); + List ret = IOUtils.readLines(conn.getInputStream(), + Charset.defaultCharset()); Assert.assertEquals(1, ret.size()); Assert.assertEquals(FOO_USER, ret.get(0)); @@ -624,7 +626,8 @@ public Void run() throws Exception { HttpURLConnection conn = aUrl.openConnection(url, token); Assert.assertEquals(HttpURLConnection.HTTP_OK, conn.getResponseCode()); - List ret = IOUtils.readLines(conn.getInputStream()); + List ret = IOUtils + .readLines(conn.getInputStream(), Charset.defaultCharset()); Assert.assertEquals(1, ret.size()); Assert.assertEquals(FOO_USER, ret.get(0)); @@ -848,14 +851,15 @@ public void testProxyUser() throws Exception { HttpURLConnection conn = (HttpURLConnection) new URL(strUrl).openConnection(); Assert.assertEquals(HttpURLConnection.HTTP_OK, conn.getResponseCode()); - List ret = IOUtils.readLines(conn.getInputStream()); + List ret = + IOUtils.readLines(conn.getInputStream(), Charset.defaultCharset()); Assert.assertEquals(1, ret.size()); Assert.assertEquals(OK_USER, ret.get(0)); strUrl = String.format("%s?user.name=%s&DOAS=%s", url.toExternalForm(), FOO_USER, OK_USER); conn = (HttpURLConnection) new URL(strUrl).openConnection(); Assert.assertEquals(HttpURLConnection.HTTP_OK, conn.getResponseCode()); - ret = IOUtils.readLines(conn.getInputStream()); + ret = IOUtils.readLines(conn.getInputStream(), Charset.defaultCharset()); Assert.assertEquals(1, ret.size()); Assert.assertEquals(OK_USER, ret.get(0)); @@ -872,7 +876,8 @@ public Void run() throws Exception { HttpURLConnection conn = aUrl.openConnection(url, token, OK_USER); Assert.assertEquals(HttpURLConnection.HTTP_OK, conn.getResponseCode()); - List ret = IOUtils.readLines(conn.getInputStream()); + List ret = IOUtils + .readLines(conn.getInputStream(), Charset.defaultCharset()); Assert.assertEquals(1, ret.size()); Assert.assertEquals(OK_USER, ret.get(0)); @@ -892,7 +897,8 @@ public Void run() throws Exception { conn = aUrl.openConnection(url, token, OK_USER); Assert.assertEquals(HttpURLConnection.HTTP_OK, conn.getResponseCode()); - ret = IOUtils.readLines(conn.getInputStream()); + ret = IOUtils + .readLines(conn.getInputStream(), Charset.defaultCharset()); Assert.assertEquals(1, ret.size()); Assert.assertEquals(FOO_USER, ret.get(0)); @@ -953,7 +959,8 @@ public Void run() throws Exception { HttpURLConnection conn = aUrl.openConnection(url, token); Assert.assertEquals(HttpURLConnection.HTTP_OK, conn.getResponseCode()); - List ret = IOUtils.readLines(conn.getInputStream()); + List ret = IOUtils + .readLines(conn.getInputStream(), Charset.defaultCharset()); Assert.assertEquals(1, ret.size()); Assert.assertEquals("remoteuser=" + FOO_USER+ ":ugi=" + FOO_USER, ret.get(0)); @@ -962,7 +969,8 @@ public Void run() throws Exception { conn = aUrl.openConnection(url, token, OK_USER); Assert.assertEquals(HttpURLConnection.HTTP_OK, conn.getResponseCode()); - ret = IOUtils.readLines(conn.getInputStream()); + ret = IOUtils + .readLines(conn.getInputStream(), Charset.defaultCharset()); Assert.assertEquals(1, ret.size()); Assert.assertEquals("realugi=" + FOO_USER +":remoteuser=" + OK_USER + ":ugi=" + OK_USER, ret.get(0)); @@ -1014,7 +1022,8 @@ public Void run() throws Exception { HttpURLConnection conn = aUrl.openConnection(url, token, OK_USER); Assert.assertEquals(HttpURLConnection.HTTP_OK, conn.getResponseCode()); - List ret = IOUtils.readLines(conn.getInputStream()); + List ret = IOUtils + .readLines(conn.getInputStream(), Charset.defaultCharset()); Assert.assertEquals(1, ret.size()); Assert.assertEquals("realugi=" + FOO_USER +":remoteuser=" + OK_USER + ":ugi=" + OK_USER, ret.get(0)); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestApplicationClassLoader.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestApplicationClassLoader.java index 4be74ba8e7..ae656742e9 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestApplicationClassLoader.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestApplicationClassLoader.java @@ -31,6 +31,7 @@ import java.io.IOException; import java.io.InputStream; import java.net.URL; +import java.nio.charset.Charset; import java.util.List; import java.util.jar.JarOutputStream; import java.util.zip.ZipEntry; @@ -134,7 +135,7 @@ public void testGetResource() throws IOException { InputStream in = appClassloader.getResourceAsStream("resource.txt"); assertNotNull("Resource should not be null for app classloader", in); - assertEquals("hello", IOUtils.toString(in)); + assertEquals("hello", IOUtils.toString(in, Charset.defaultCharset())); } private File makeTestJar() throws IOException { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/CancelCommand.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/CancelCommand.java index 4e7b38c2f3..0789430a38 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/CancelCommand.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/CancelCommand.java @@ -32,6 +32,7 @@ import org.apache.hadoop.hdfs.tools.DiskBalancerCLI; import java.io.IOException; +import java.nio.charset.Charset; /** * Cancels a running plan. @@ -76,7 +77,7 @@ public void execute(CommandLine cmd) throws Exception { "Invalid plan file specified."); String planData = null; try (FSDataInputStream plan = open(planFile)) { - planData = IOUtils.toString(plan); + planData = IOUtils.toString(plan, Charset.defaultCharset()); } cancelPlan(planData); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/ExecuteCommand.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/ExecuteCommand.java index 7eac881138..4cc1ade5d0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/ExecuteCommand.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/ExecuteCommand.java @@ -32,7 +32,7 @@ import org.apache.hadoop.hdfs.tools.DiskBalancerCLI; import java.io.IOException; - +import java.nio.charset.Charset; /** * executes a given plan. @@ -69,7 +69,7 @@ public void execute(CommandLine cmd) throws Exception { String planData = null; try (FSDataInputStream plan = open(planFile)) { - planData = IOUtils.toString(plan); + planData = IOUtils.toString(plan, Charset.defaultCharset()); } boolean skipDateCheck = false; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/shell/TestHdfsTextCommand.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/shell/TestHdfsTextCommand.java index 23de658529..cb7773c838 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/shell/TestHdfsTextCommand.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/shell/TestHdfsTextCommand.java @@ -25,6 +25,7 @@ import java.io.InputStream; import java.io.StringWriter; import java.lang.reflect.Method; +import java.nio.charset.Charset; import org.apache.commons.io.IOUtils; import org.apache.hadoop.conf.Configuration; @@ -106,7 +107,7 @@ public void testDisplayForAvroFiles() throws Exception { private String inputStreamToString(InputStream stream) throws IOException { StringWriter writer = new StringWriter(); - IOUtils.copy(stream, writer); + IOUtils.copy(stream, writer, Charset.defaultCharset()); return writer.toString(); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewerForXAttr.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewerForXAttr.java index 74069b0e48..f402017b9c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewerForXAttr.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewerForXAttr.java @@ -25,6 +25,7 @@ import java.net.HttpURLConnection; import java.net.URI; import java.net.URL; +import java.nio.charset.Charset; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -125,7 +126,8 @@ public void testWebImageViewerForListXAttrs() throws Exception { assertEquals(HttpURLConnection.HTTP_OK, connection.getResponseCode()); - String content = IOUtils.toString(connection.getInputStream()); + String content = IOUtils + .toString(connection.getInputStream(), Charset.defaultCharset()); assertTrue("Missing user.attr1 in response ", content.contains("user.attr1")); @@ -150,7 +152,8 @@ public void testWebImageViewerForGetXAttrsWithOutParameters() connection.connect(); assertEquals(HttpURLConnection.HTTP_OK, connection.getResponseCode()); - String content = IOUtils.toString(connection.getInputStream()); + String content = IOUtils + .toString(connection.getInputStream(), Charset.defaultCharset()); assertTrue("Missing user.attr1 in response ", content.contains("user.attr1")); @@ -183,7 +186,8 @@ public void testWebImageViewerForGetXAttrsWithParameters() throws Exception { connection.connect(); assertEquals(HttpURLConnection.HTTP_OK, connection.getResponseCode()); - String content = IOUtils.toString(connection.getInputStream()); + String content = IOUtils + .toString(connection.getInputStream(), Charset.defaultCharset()); assertEquals(attr1JSon, content); } } @@ -205,7 +209,8 @@ public void testWebImageViewerForGetXAttrsWithCodecParameters() connection.connect(); assertEquals(HttpURLConnection.HTTP_OK, connection.getResponseCode()); - String content = IOUtils.toString(connection.getInputStream()); + String content = IOUtils + .toString(connection.getInputStream(), Charset.defaultCharset()); assertEquals(attr1JSon, content); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java index 68087c4c4e..5322960a3c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java @@ -48,6 +48,7 @@ import java.net.URI; import java.net.URISyntaxException; import java.net.URL; +import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; import java.security.PrivilegedExceptionAction; import java.util.Arrays; @@ -1552,7 +1553,8 @@ private void checkResponseContainsLocation(URL url, String TYPE) HttpURLConnection conn = (HttpURLConnection) url.openConnection(); conn.setRequestMethod(TYPE); conn.setInstanceFollowRedirects(false); - String response = IOUtils.toString(conn.getInputStream()); + String response = + IOUtils.toString(conn.getInputStream(), Charset.defaultCharset()); LOG.info("Response was : " + response); Assert.assertEquals( "Response wasn't " + HttpURLConnection.HTTP_OK, diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestJavaSerialization.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestJavaSerialization.java index e5ef4d2687..414b3ca30d 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestJavaSerialization.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestJavaSerialization.java @@ -23,6 +23,7 @@ import java.io.OutputStream; import java.io.OutputStreamWriter; import java.io.Writer; +import java.nio.charset.Charset; import java.util.Iterator; import java.util.StringTokenizer; @@ -134,7 +135,8 @@ public void testMapReduceJob() throws Exception { new Utils.OutputFileUtils.OutputFilesFilter())); assertEquals(1, outputFiles.length); try (InputStream is = fs.open(outputFiles[0])) { - String reduceOutput = org.apache.commons.io.IOUtils.toString(is); + String reduceOutput = + org.apache.commons.io.IOUtils.toString(is, Charset.defaultCharset()); String[] lines = reduceOutput.split("\n"); assertEquals("Unexpected output; received output '" + reduceOutput + "'", "a\t1", lines[0]); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-uploader/src/test/java/org/apache/hadoop/mapred/uploader/TestFrameworkUploader.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-uploader/src/test/java/org/apache/hadoop/mapred/uploader/TestFrameworkUploader.java index 7749db1be8..1b65d17397 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-uploader/src/test/java/org/apache/hadoop/mapred/uploader/TestFrameworkUploader.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-uploader/src/test/java/org/apache/hadoop/mapred/uploader/TestFrameworkUploader.java @@ -44,6 +44,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.PrintStream; +import java.nio.charset.Charset; import java.nio.file.Files; import java.nio.file.Paths; import java.util.HashMap; @@ -436,7 +437,8 @@ public void testNativeIO() throws IOException { // Create a target file File targetFile = new File(parent, "a.txt"); try(FileOutputStream os = new FileOutputStream(targetFile)) { - IOUtils.writeLines(Lists.newArrayList("a", "b"), null, os); + IOUtils.writeLines(Lists.newArrayList("a", "b"), null, os, + Charset.defaultCharset()); } Assert.assertFalse(uploader.checkSymlink(targetFile)); diff --git a/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/resourcegz/ResourceGzMojo.java b/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/resourcegz/ResourceGzMojo.java index 0b0139eebe..359389fddb 100644 --- a/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/resourcegz/ResourceGzMojo.java +++ b/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/resourcegz/ResourceGzMojo.java @@ -24,6 +24,7 @@ import java.io.File; import java.io.FileOutputStream; import java.io.IOException; +import java.nio.charset.Charset; import java.nio.file.Files; import java.nio.file.Path; import java.util.Arrays; @@ -113,7 +114,7 @@ public void accept(Path path) { BufferedReader is = Files.newBufferedReader(path) ) { getLog().info("Compressing " + path + " to " + outFile); - IOUtils.copy(is, os); + IOUtils.copy(is, os, Charset.defaultCharset()); } } else { throw new IOException("Directory " + outFile.getParent() diff --git a/hadoop-tools/hadoop-archive-logs/src/test/java/org/apache/hadoop/tools/TestHadoopArchiveLogs.java b/hadoop-tools/hadoop-archive-logs/src/test/java/org/apache/hadoop/tools/TestHadoopArchiveLogs.java index a1b662ce18..e324f1dedd 100644 --- a/hadoop-tools/hadoop-archive-logs/src/test/java/org/apache/hadoop/tools/TestHadoopArchiveLogs.java +++ b/hadoop-tools/hadoop-archive-logs/src/test/java/org/apache/hadoop/tools/TestHadoopArchiveLogs.java @@ -42,6 +42,7 @@ import java.io.File; import java.io.IOException; +import java.nio.charset.Charset; import java.util.Random; public class TestHadoopArchiveLogs { @@ -278,7 +279,8 @@ private void _testGenerateScript(boolean proxy) throws Exception { Assert.assertFalse(localScript.exists()); hal.generateScript(localScript); Assert.assertTrue(localScript.exists()); - String script = IOUtils.toString(localScript.toURI()); + String script = + IOUtils.toString(localScript.toURI(), Charset.defaultCharset()); String[] lines = script.split("\n"); Assert.assertEquals(22, lines.length); Assert.assertEquals("#!/bin/bash", lines[0]); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java index fa94491ec4..1bbf0f2c15 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java @@ -97,6 +97,7 @@ import java.net.URI; import java.net.URISyntaxException; import java.nio.ByteBuffer; +import java.nio.charset.Charset; import java.text.MessageFormat; import java.util.Collection; import java.util.HashMap; @@ -548,7 +549,7 @@ private void createConfigFileCache(final FileSystem fileSystem) { case TEMPLATE: try (FSDataInputStream fileInput = fileSystem .open(new Path(key.getSrcFile()))) { - return IOUtils.toString(fileInput); + return IOUtils.toString(fileInput, Charset.defaultCharset()); } default: return null; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java index 0b091e2241..f7c0a7b374 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java @@ -48,6 +48,7 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.io.OutputStream; +import java.nio.charset.Charset; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -465,7 +466,8 @@ private static void resolvePlainTemplateAndSaveOnHdfs(FileSystem fs, content = substituteStrWithTokens(content, tokensForSubstitution); try (OutputStream output = fs.create(remoteFile)) { - org.apache.commons.io.IOUtils.write(content, output); + org.apache.commons.io.IOUtils + .write(content, output, Charset.defaultCharset()); } catch (IOException e) { log.info("Failed to create " + remoteFile); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/DockerClientConfigHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/DockerClientConfigHandler.java index ac2ce2c049..bb6660e074 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/DockerClientConfigHandler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/DockerClientConfigHandler.java @@ -87,7 +87,7 @@ public static Credentials readCredentialsFromConfigFile(Path configFile, if (fs != null) { FSDataInputStream fileHandle = fs.open(configFile); if (fileHandle != null) { - contents = IOUtils.toString(fileHandle); + contents = IOUtils.toString(fileHandle, Charset.defaultCharset()); } } if (contents == null) {