big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-17950. Provide replacement for deprecated APIs of commons-io IOUtils (#3515)

Browse Source 
Signed-off-by: Akira Ajisaka <aajisaka@apache.org>
This commit is contained in:
Viraj Jasani 2021-10-07 07:28:29 +05:30 committed by GitHub
parent c36f9402dc
commit 8071dbb9c6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
18 changed files with 64 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/KDiag.java
View File

@ -46,6 +46,7 @@
import java.io.PrintWriter;
import java.lang.reflect.InvocationTargetException;
import java.net.InetAddress;
import java.nio.charset.Charset;
import java.nio.file.Files;
import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
@ -923,7 +924,7 @@ private void printEnv(String variable) {
*/
private void dump(File file) throws IOException {
try (InputStream in = Files.newInputStream(file.toPath())) {
for (String line : IOUtils.readLines(in)) {
for (String line : IOUtils.readLines(in, Charset.defaultCharset())) {
println("%s", line);
}
}

4
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ProviderUtils.java
View File

@ -23,6 +23,7 @@
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
import java.nio.charset.Charset;
import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting;
import org.apache.commons.io.IOUtils;
@ -224,7 +225,8 @@ public static char[] locatePassword(String envWithPass, String fileWithPass)
throw new IOException("Password file does not exist");
}
try (InputStream is = pwdFile.openStream()) {
pass = IOUtils.toString(is).trim().toCharArray();
pass = IOUtils.toString(is, Charset.defaultCharset()).trim()
.toCharArray();
}
}
}

3
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestTextCommand.java
View File

@ -27,6 +27,7 @@
import java.io.StringWriter;
import java.lang.reflect.Method;
import java.net.URI;
import java.nio.charset.Charset;
import java.nio.file.Files;
import java.nio.file.Paths;
@ -124,7 +125,7 @@ public InputStream getInputStream(PathData item) throws IOException {
private String inputStreamToString(InputStream stream) throws IOException {
StringWriter writer = new StringWriter();
IOUtils.copy(stream, writer);
IOUtils.copy(stream, writer, Charset.defaultCharset());
return writer.toString();
}

3
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestKDiag.java
View File

@ -36,6 +36,7 @@
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.nio.charset.Charset;
import java.util.Properties;
import java.util.concurrent.TimeUnit;
@ -235,7 +236,7 @@ public void testKeytabUnknownPrincipal() throws Throwable {
*/
private void dump(File file) throws IOException {
try (FileInputStream in = new FileInputStream(file)) {
for (String line : IOUtils.readLines(in)) {
for (String line : IOUtils.readLines(in, Charset.defaultCharset())) {
LOG.info(line);
}
}

27
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/web/TestWebDelegationToken.java
View File

@ -64,6 +64,7 @@
import java.io.Writer;
import java.net.HttpURLConnection;
import java.net.URL;
import java.nio.charset.Charset;
import java.security.Principal;
import java.security.PrivilegedActionException;
import java.security.PrivilegedExceptionAction;
@ -554,7 +555,8 @@ public Void run() throws Exception {
HttpURLConnection conn = aUrl.openConnection(url, token);
Assert.assertEquals(HttpURLConnection.HTTP_OK,
conn.getResponseCode());
List<String> ret = IOUtils.readLines(conn.getInputStream());
List<String> ret = IOUtils.readLines(conn.getInputStream(),
Charset.defaultCharset());
Assert.assertEquals(1, ret.size());
Assert.assertEquals(FOO_USER, ret.get(0));
@ -624,7 +626,8 @@ public Void run() throws Exception {
HttpURLConnection conn = aUrl.openConnection(url, token);
Assert.assertEquals(HttpURLConnection.HTTP_OK,
conn.getResponseCode());
List<String> ret = IOUtils.readLines(conn.getInputStream());
List<String> ret = IOUtils
.readLines(conn.getInputStream(), Charset.defaultCharset());
Assert.assertEquals(1, ret.size());
Assert.assertEquals(FOO_USER, ret.get(0));
@ -848,14 +851,15 @@ public void testProxyUser() throws Exception {
HttpURLConnection conn =
(HttpURLConnection) new URL(strUrl).openConnection();
Assert.assertEquals(HttpURLConnection.HTTP_OK, conn.getResponseCode());
List<String> ret = IOUtils.readLines(conn.getInputStream());
List<String> ret =
IOUtils.readLines(conn.getInputStream(), Charset.defaultCharset());
Assert.assertEquals(1, ret.size());
Assert.assertEquals(OK_USER, ret.get(0));
strUrl = String.format("%s?user.name=%s&DOAS=%s", url.toExternalForm(),
FOO_USER, OK_USER);
conn = (HttpURLConnection) new URL(strUrl).openConnection();
Assert.assertEquals(HttpURLConnection.HTTP_OK, conn.getResponseCode());
ret = IOUtils.readLines(conn.getInputStream());
ret = IOUtils.readLines(conn.getInputStream(), Charset.defaultCharset());
Assert.assertEquals(1, ret.size());
Assert.assertEquals(OK_USER, ret.get(0));
@ -872,7 +876,8 @@ public Void run() throws Exception {
HttpURLConnection conn = aUrl.openConnection(url, token, OK_USER);
Assert.assertEquals(HttpURLConnection.HTTP_OK,
conn.getResponseCode());
List<String> ret = IOUtils.readLines(conn.getInputStream());
List<String> ret = IOUtils
.readLines(conn.getInputStream(), Charset.defaultCharset());
Assert.assertEquals(1, ret.size());
Assert.assertEquals(OK_USER, ret.get(0));
@ -892,7 +897,8 @@ public Void run() throws Exception {
conn = aUrl.openConnection(url, token, OK_USER);
Assert.assertEquals(HttpURLConnection.HTTP_OK,
conn.getResponseCode());
ret = IOUtils.readLines(conn.getInputStream());
ret = IOUtils
.readLines(conn.getInputStream(), Charset.defaultCharset());
Assert.assertEquals(1, ret.size());
Assert.assertEquals(FOO_USER, ret.get(0));
@ -953,7 +959,8 @@ public Void run() throws Exception {
HttpURLConnection conn = aUrl.openConnection(url, token);
Assert.assertEquals(HttpURLConnection.HTTP_OK,
conn.getResponseCode());
List<String> ret = IOUtils.readLines(conn.getInputStream());
List<String> ret = IOUtils
.readLines(conn.getInputStream(), Charset.defaultCharset());
Assert.assertEquals(1, ret.size());
Assert.assertEquals("remoteuser=" + FOO_USER+ ":ugi=" + FOO_USER,
ret.get(0));
@ -962,7 +969,8 @@ public Void run() throws Exception {
conn = aUrl.openConnection(url, token, OK_USER);
Assert.assertEquals(HttpURLConnection.HTTP_OK,
conn.getResponseCode());
ret = IOUtils.readLines(conn.getInputStream());
ret = IOUtils
.readLines(conn.getInputStream(), Charset.defaultCharset());
Assert.assertEquals(1, ret.size());
Assert.assertEquals("realugi=" + FOO_USER +":remoteuser=" + OK_USER +
":ugi=" + OK_USER, ret.get(0));
@ -1014,7 +1022,8 @@ public Void run() throws Exception {
HttpURLConnection conn = aUrl.openConnection(url, token, OK_USER);
Assert.assertEquals(HttpURLConnection.HTTP_OK,
conn.getResponseCode());
List<String> ret = IOUtils.readLines(conn.getInputStream());
List<String> ret = IOUtils
.readLines(conn.getInputStream(), Charset.defaultCharset());
Assert.assertEquals(1, ret.size());
Assert.assertEquals("realugi=" + FOO_USER +":remoteuser=" + OK_USER +
":ugi=" + OK_USER, ret.get(0));

3
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestApplicationClassLoader.java
View File

@ -31,6 +31,7 @@
import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
import java.nio.charset.Charset;
import java.util.List;
import java.util.jar.JarOutputStream;
import java.util.zip.ZipEntry;
@ -134,7 +135,7 @@ public void testGetResource() throws IOException {
InputStream in = appClassloader.getResourceAsStream("resource.txt");
assertNotNull("Resource should not be null for app classloader", in);
assertEquals("hello", IOUtils.toString(in));
assertEquals("hello", IOUtils.toString(in, Charset.defaultCharset()));
}
private File makeTestJar() throws IOException {

3
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/CancelCommand.java
View File

@ -32,6 +32,7 @@
import org.apache.hadoop.hdfs.tools.DiskBalancerCLI;
import java.io.IOException;
import java.nio.charset.Charset;
/**
* Cancels a running plan.
@ -76,7 +77,7 @@ public void execute(CommandLine cmd) throws Exception {
"Invalid plan file specified.");
String planData = null;
try (FSDataInputStream plan = open(planFile)) {
planData = IOUtils.toString(plan);
planData = IOUtils.toString(plan, Charset.defaultCharset());
}
cancelPlan(planData);
}

4
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/ExecuteCommand.java
View File

@ -32,7 +32,7 @@
import org.apache.hadoop.hdfs.tools.DiskBalancerCLI;
import java.io.IOException;
import java.nio.charset.Charset;
/**
* executes a given plan.
@ -69,7 +69,7 @@ public void execute(CommandLine cmd) throws Exception {
String planData = null;
try (FSDataInputStream plan = open(planFile)) {
planData = IOUtils.toString(plan);
planData = IOUtils.toString(plan, Charset.defaultCharset());
}
boolean skipDateCheck = false;

3
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/shell/TestHdfsTextCommand.java
View File

@ -25,6 +25,7 @@
import java.io.InputStream;
import java.io.StringWriter;
import java.lang.reflect.Method;
import java.nio.charset.Charset;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
@ -106,7 +107,7 @@ public void testDisplayForAvroFiles() throws Exception {
private String inputStreamToString(InputStream stream) throws IOException {
StringWriter writer = new StringWriter();
IOUtils.copy(stream, writer);
IOUtils.copy(stream, writer, Charset.defaultCharset());
return writer.toString();
}

13
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewerForXAttr.java
View File

@ -25,6 +25,7 @@
import java.net.HttpURLConnection;
import java.net.URI;
import java.net.URL;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@ -125,7 +126,8 @@ public void testWebImageViewerForListXAttrs() throws Exception {
assertEquals(HttpURLConnection.HTTP_OK, connection.getResponseCode());
String content = IOUtils.toString(connection.getInputStream());
String content = IOUtils
.toString(connection.getInputStream(), Charset.defaultCharset());
assertTrue("Missing user.attr1 in response ",
content.contains("user.attr1"));
@ -150,7 +152,8 @@ public void testWebImageViewerForGetXAttrsWithOutParameters()
connection.connect();
assertEquals(HttpURLConnection.HTTP_OK, connection.getResponseCode());
String content = IOUtils.toString(connection.getInputStream());
String content = IOUtils
.toString(connection.getInputStream(), Charset.defaultCharset());
assertTrue("Missing user.attr1 in response ",
content.contains("user.attr1"));
@ -183,7 +186,8 @@ public void testWebImageViewerForGetXAttrsWithParameters() throws Exception {
connection.connect();
assertEquals(HttpURLConnection.HTTP_OK, connection.getResponseCode());
String content = IOUtils.toString(connection.getInputStream());
String content = IOUtils
.toString(connection.getInputStream(), Charset.defaultCharset());
assertEquals(attr1JSon, content);
}
}
@ -205,7 +209,8 @@ public void testWebImageViewerForGetXAttrsWithCodecParameters()
connection.connect();
assertEquals(HttpURLConnection.HTTP_OK, connection.getResponseCode());
String content = IOUtils.toString(connection.getInputStream());
String content = IOUtils
.toString(connection.getInputStream(), Charset.defaultCharset());
assertEquals(attr1JSon, content);
}

4
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java
View File

@ -48,6 +48,7 @@
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.security.PrivilegedExceptionAction;
import java.util.Arrays;
@ -1552,7 +1553,8 @@ private void checkResponseContainsLocation(URL url, String TYPE)
HttpURLConnection conn = (HttpURLConnection) url.openConnection();
conn.setRequestMethod(TYPE);
conn.setInstanceFollowRedirects(false);
String response = IOUtils.toString(conn.getInputStream());
String response =
IOUtils.toString(conn.getInputStream(), Charset.defaultCharset());
LOG.info("Response was : " + response);
Assert.assertEquals(
"Response wasn't " + HttpURLConnection.HTTP_OK,

4
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestJavaSerialization.java
View File

@ -23,6 +23,7 @@
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.Writer;
import java.nio.charset.Charset;
import java.util.Iterator;
import java.util.StringTokenizer;
@ -134,7 +135,8 @@ public void testMapReduceJob() throws Exception {
new Utils.OutputFileUtils.OutputFilesFilter()));
assertEquals(1, outputFiles.length);
try (InputStream is = fs.open(outputFiles[0])) {
String reduceOutput = org.apache.commons.io.IOUtils.toString(is);
String reduceOutput =
org.apache.commons.io.IOUtils.toString(is, Charset.defaultCharset());
String[] lines = reduceOutput.split("\n");
assertEquals("Unexpected output; received output '" + reduceOutput + "'",
"a\t1", lines[0]);

4
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-uploader/src/test/java/org/apache/hadoop/mapred/uploader/TestFrameworkUploader.java
View File

@ -44,6 +44,7 @@
import java.io.IOException;
import java.io.InputStream;
import java.io.PrintStream;
import java.nio.charset.Charset;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.HashMap;
@ -436,7 +437,8 @@ public void testNativeIO() throws IOException {
// Create a target file
File targetFile = new File(parent, "a.txt");
try(FileOutputStream os = new FileOutputStream(targetFile)) {
IOUtils.writeLines(Lists.newArrayList("a", "b"), null, os);
IOUtils.writeLines(Lists.newArrayList("a", "b"), null, os,
Charset.defaultCharset());
}
Assert.assertFalse(uploader.checkSymlink(targetFile));

3
hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/resourcegz/ResourceGzMojo.java
View File

@ -24,6 +24,7 @@
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.nio.charset.Charset;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Arrays;
@ -113,7 +114,7 @@ public void accept(Path path) {
BufferedReader is = Files.newBufferedReader(path)
) {
getLog().info("Compressing " + path + " to " + outFile);
IOUtils.copy(is, os);
IOUtils.copy(is, os, Charset.defaultCharset());
}
} else {
throw new IOException("Directory " + outFile.getParent()

4
hadoop-tools/hadoop-archive-logs/src/test/java/org/apache/hadoop/tools/TestHadoopArchiveLogs.java
View File

@ -42,6 +42,7 @@
import java.io.File;
import java.io.IOException;
import java.nio.charset.Charset;
import java.util.Random;
public class TestHadoopArchiveLogs {
@ -278,7 +279,8 @@ private void _testGenerateScript(boolean proxy) throws Exception {
Assert.assertFalse(localScript.exists());
hal.generateScript(localScript);
Assert.assertTrue(localScript.exists());
String script = IOUtils.toString(localScript.toURI());
String script =
IOUtils.toString(localScript.toURI(), Charset.defaultCharset());
String[] lines = script.split("\n");
Assert.assertEquals(22, lines.length);
Assert.assertEquals("#!/bin/bash", lines[0]);

3
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
View File

@ -97,6 +97,7 @@
import java.net.URI;
import java.net.URISyntaxException;
import java.nio.ByteBuffer;
import java.nio.charset.Charset;
import java.text.MessageFormat;
import java.util.Collection;
import java.util.HashMap;
@ -548,7 +549,7 @@ private void createConfigFileCache(final FileSystem fileSystem) {
case TEMPLATE:
try (FSDataInputStream fileInput = fileSystem
.open(new Path(key.getSrcFile()))) {
return IOUtils.toString(fileInput);
return IOUtils.toString(fileInput, Charset.defaultCharset());
}
default:
return null;

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java
View File

@ -48,6 +48,7 @@
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.OutputStream;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
@ -465,7 +466,8 @@ private static void resolvePlainTemplateAndSaveOnHdfs(FileSystem fs,
content = substituteStrWithTokens(content, tokensForSubstitution);
try (OutputStream output = fs.create(remoteFile)) {
org.apache.commons.io.IOUtils.write(content, output);
org.apache.commons.io.IOUtils
.write(content, output, Charset.defaultCharset());
} catch (IOException e) {
log.info("Failed to create " + remoteFile);
}

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/DockerClientConfigHandler.java
View File

@ -87,7 +87,7 @@ public static Credentials readCredentialsFromConfigFile(Path configFile,
if (fs != null) {
FSDataInputStream fileHandle = fs.open(configFile);
if (fileHandle != null) {
contents = IOUtils.toString(fileHandle);
contents = IOUtils.toString(fileHandle, Charset.defaultCharset());
}
}
if (contents == null) {