big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-11100. Support to configure ftpClient.setControlKeepAliveTimeout.

Browse Source 
Contributed by Adam Antal.

Signed-off-by: Xiao Chen <xiao@apache.org>
This commit is contained in:
Adam Antal 2018-10-17 11:32:17 -07:00 committed by Xiao Chen
parent d54f5598f4
commit 24dc068a36
4 changed files with 37 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FTPFileSystem.java
View File

@ -62,6 +62,7 @@ public class FTPFileSystem extends FileSystem {
public static final int DEFAULT_BUFFER_SIZE = 1024 * 1024;
public static final int DEFAULT_BLOCK_SIZE = 4 * 1024;
public static final long DEFAULT_TIMEOUT = 0;
public static final String FS_FTP_USER_PREFIX = "fs.ftp.user.";
public static final String FS_FTP_HOST = "fs.ftp.host";
public static final String FS_FTP_HOST_PORT = "fs.ftp.host.port";
@ -71,6 +72,7 @@ public class FTPFileSystem extends FileSystem {
public static final String FS_FTP_TRANSFER_MODE = "fs.ftp.transfer.mode";
public static final String E_SAME_DIRECTORY_ONLY =
"only same directory renames are supported";
public static final String FS_FTP_TIMEOUT = "fs.ftp.timeout";
private URI uri;
@ -150,6 +152,7 @@ private FTPClient connect() throws IOException {
client.setFileTransferMode(getTransferMode(conf));
client.setFileType(FTP.BINARY_FILE_TYPE);
client.setBufferSize(DEFAULT_BUFFER_SIZE);
setTimeout(client, conf);
setDataConnectionMode(client, conf);
} else {
throw new IOException("Login failed on server - " + host + ", port - "
@ -159,6 +162,16 @@ private FTPClient connect() throws IOException {
return client;
}
/**
* Set the FTPClient's timeout based on configuration.
* FS_FTP_TIMEOUT is set as timeout (defaults to DEFAULT_TIMEOUT).
*/
@VisibleForTesting
void setTimeout(FTPClient client, Configuration conf) {
long timeout = conf.getLong(FS_FTP_TIMEOUT, DEFAULT_TIMEOUT);
client.setControlKeepAliveTimeout(timeout);
}
/**
* Set FTP's transfer mode based on configuration. Valid values are
* STREAM_TRANSFER_MODE, BLOCK_TRANSFER_MODE and COMPRESSED_TRANSFER_MODE.

8
hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
View File

@ -925,6 +925,14 @@
</description>
</property>
<property>
<name>fs.ftp.timeout</name>
<value>0</value>
<description>
FTP filesystem's timeout in seconds.
</description>
</property>
<property>
<name>fs.df.interval</name>
<value>60000</value>

1
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java
View File

@ -93,6 +93,7 @@ public void initializeMemberVariables() {
xmlPropsToSkipCompare.add("fs.ftp.user.localhost");
xmlPropsToSkipCompare.add("fs.ftp.data.connection.mode");
xmlPropsToSkipCompare.add("fs.ftp.transfer.mode");
xmlPropsToSkipCompare.add("fs.ftp.timeout");
xmlPropsToSkipCompare.add("hadoop.tmp.dir");
xmlPropsToSkipCompare.add("nfs3.mountd.port");
xmlPropsToSkipCompare.add("nfs3.server.port");

15
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/ftp/TestFTPFileSystem.java
View File

@ -137,4 +137,19 @@ private FTPFile getFTPFileOf(int access, FsAction action) {
return ftpFile;
}
@Test
public void testFTPSetTimeout() {
Configuration conf = new Configuration();
FTPClient client = new FTPClient();
FTPFileSystem ftp = new FTPFileSystem();
ftp.setTimeout(client, conf);
assertEquals(client.getControlKeepAliveTimeout(),
FTPFileSystem.DEFAULT_TIMEOUT);
long timeout = 600;
conf.setLong(FTPFileSystem.FS_FTP_TIMEOUT, timeout);
ftp.setTimeout(client, conf);
assertEquals(client.getControlKeepAliveTimeout(), timeout);
}
}