big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-16182. Update abfs storage back-end with "close" flag when application is done writing to a file.

Browse Source 
Contributed by Vishwajeet Dusane.
This commit is contained in:
Vishwajeet Dusane 2019-03-18 13:18:08 +00:00 committed by Steve Loughran
parent cb4d911a82
commit 1edf1914ac
No known key found for this signature in database
GPG Key ID: D22CF846DBB162A0
3 changed files with 14 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/HttpQueryParams.java
View File

@ -35,6 +35,7 @@ public final class HttpQueryParams {
public static final String QUERY_PARAM_POSITION = "position";
public static final String QUERY_PARAM_TIMEOUT = "timeout";
public static final String QUERY_PARAM_RETAIN_UNCOMMITTED_DATA = "retainUncommittedData";
public static final String QUERY_PARAM_CLOSE = "close";
public static final String QUERY_PARAM_UPN = "upn";
private HttpQueryParams() {}

3
hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java
View File

@ -300,7 +300,7 @@ public AbfsRestOperation append(final String path, final long position, final by
return op;
}
public AbfsRestOperation flush(final String path, final long position, boolean retainUncommittedData)
public AbfsRestOperation flush(final String path, final long position, boolean retainUncommittedData, boolean isClose)
throws AzureBlobFileSystemException {
final List<AbfsHttpHeader> requestHeaders = createDefaultHeaders();
// JDK7 does not support PATCH, so to workaround the issue we will use
@ -312,6 +312,7 @@ public AbfsRestOperation flush(final String path, final long position, boolean r
abfsUriQueryBuilder.addQuery(QUERY_PARAM_ACTION, FLUSH_ACTION);
abfsUriQueryBuilder.addQuery(QUERY_PARAM_POSITION, Long.toString(position));
abfsUriQueryBuilder.addQuery(QUERY_PARAM_RETAIN_UNCOMMITTED_DATA, String.valueOf(retainUncommittedData));
abfsUriQueryBuilder.addQuery(QUERY_PARAM_CLOSE, String.valueOf(isClose));
final URL url = createRequestUrl(path, abfsUriQueryBuilder.toString());
final AbfsRestOperation op = new AbfsRestOperation(

21
hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsOutputStream.java
View File

@ -200,7 +200,7 @@ public void flush() throws IOException {
@Override
public void hsync() throws IOException {
if (supportFlush) {
flushInternal();
flushInternal(false);
}
}
@ -211,7 +211,7 @@ public void hsync() throws IOException {
@Override
public void hflush() throws IOException {
if (supportFlush) {
flushInternal();
flushInternal(false);
}
}
@ -230,7 +230,7 @@ public synchronized void close() throws IOException {
}
try {
flushInternal();
flushInternal(true);
threadExecutor.shutdown();
} finally {
lastError = new IOException(FSExceptionMessages.STREAM_IS_CLOSED);
@ -244,10 +244,10 @@ public synchronized void close() throws IOException {
}
}
private synchronized void flushInternal() throws IOException {
private synchronized void flushInternal(boolean isClose) throws IOException {
maybeThrowLastError();
writeCurrentBufferToService();
flushWrittenBytesToService();
flushWrittenBytesToService(isClose);
}
private synchronized void flushInternalAsync() throws IOException {
@ -288,7 +288,7 @@ public Void call() throws Exception {
shrinkWriteOperationQueue();
}
private synchronized void flushWrittenBytesToService() throws IOException {
private synchronized void flushWrittenBytesToService(boolean isClose) throws IOException {
for (WriteOperation writeOperation : writeOperations) {
try {
writeOperation.task.get();
@ -306,21 +306,22 @@ private synchronized void flushWrittenBytesToService() throws IOException {
throw lastError;
}
}
flushWrittenBytesToServiceInternal(position, false);
flushWrittenBytesToServiceInternal(position, false, isClose);
}
private synchronized void flushWrittenBytesToServiceAsync() throws IOException {
shrinkWriteOperationQueue();
if (this.lastTotalAppendOffset > this.lastFlushOffset) {
this.flushWrittenBytesToServiceInternal(this.lastTotalAppendOffset, true);
this.flushWrittenBytesToServiceInternal(this.lastTotalAppendOffset, true,
false/*Async flush on close not permitted*/);
}
}
private synchronized void flushWrittenBytesToServiceInternal(final long offset,
final boolean retainUncommitedData) throws IOException {
final boolean retainUncommitedData, final boolean isClose) throws IOException {
try {
client.flush(path, offset, retainUncommitedData);
client.flush(path, offset, retainUncommitedData, isClose);
} catch (AzureBlobFileSystemException ex) {
if (ex instanceof AbfsRestOperationException) {
if (((AbfsRestOperationException) ex).getStatusCode() == HttpURLConnection.HTTP_NOT_FOUND) {