big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-16182. Update abfs storage back-end with "close" flag when application is done writing to a file.

Browse Source 
Contributed by Vishwajeet Dusane.
This commit is contained in:
Vishwajeet Dusane 2019-03-18 13:18:08 +00:00 committed by Steve Loughran
parent cb4d911a82
commit 1edf1914ac
No known key found for this signature in database
GPG Key ID: D22CF846DBB162A0
3 changed files with 14 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/HttpQueryParams.java
View File

@ -35,6 +35,7 @@ public final class HttpQueryParams {
public static final String QUERY_PARAM_POSITION = "position"; public static final String QUERY_PARAM_POSITION = "position";
public static final String QUERY_PARAM_TIMEOUT = "timeout"; public static final String QUERY_PARAM_TIMEOUT = "timeout";
public static final String QUERY_PARAM_RETAIN_UNCOMMITTED_DATA = "retainUncommittedData"; public static final String QUERY_PARAM_RETAIN_UNCOMMITTED_DATA = "retainUncommittedData";
public static final String QUERY_PARAM_CLOSE = "close";
public static final String QUERY_PARAM_UPN = "upn"; public static final String QUERY_PARAM_UPN = "upn";
private HttpQueryParams() {} private HttpQueryParams() {}

3
hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java
View File

@ -300,7 +300,7 @@ public AbfsRestOperation append(final String path, final long position, final by
return op; return op;
} }
public AbfsRestOperation flush(final String path, final long position, boolean retainUncommittedData) public AbfsRestOperation flush(final String path, final long position, boolean retainUncommittedData, boolean isClose)
throws AzureBlobFileSystemException { throws AzureBlobFileSystemException {
final List<AbfsHttpHeader> requestHeaders = createDefaultHeaders(); final List<AbfsHttpHeader> requestHeaders = createDefaultHeaders();
// JDK7 does not support PATCH, so to workaround the issue we will use // JDK7 does not support PATCH, so to workaround the issue we will use
@ -312,6 +312,7 @@ public AbfsRestOperation flush(final String path, final long position, boolean r
abfsUriQueryBuilder.addQuery(QUERY_PARAM_ACTION, FLUSH_ACTION); abfsUriQueryBuilder.addQuery(QUERY_PARAM_ACTION, FLUSH_ACTION);
abfsUriQueryBuilder.addQuery(QUERY_PARAM_POSITION, Long.toString(position)); abfsUriQueryBuilder.addQuery(QUERY_PARAM_POSITION, Long.toString(position));
abfsUriQueryBuilder.addQuery(QUERY_PARAM_RETAIN_UNCOMMITTED_DATA, String.valueOf(retainUncommittedData)); abfsUriQueryBuilder.addQuery(QUERY_PARAM_RETAIN_UNCOMMITTED_DATA, String.valueOf(retainUncommittedData));
abfsUriQueryBuilder.addQuery(QUERY_PARAM_CLOSE, String.valueOf(isClose));
final URL url = createRequestUrl(path, abfsUriQueryBuilder.toString()); final URL url = createRequestUrl(path, abfsUriQueryBuilder.toString());
final AbfsRestOperation op = new AbfsRestOperation( final AbfsRestOperation op = new AbfsRestOperation(

21
hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsOutputStream.java
View File

@ -200,7 +200,7 @@ public void flush() throws IOException {
@Override @Override
public void hsync() throws IOException { public void hsync() throws IOException {
if (supportFlush) { if (supportFlush) {
flushInternal(); flushInternal(false);
} }
} }
@ -211,7 +211,7 @@ public void hsync() throws IOException {
@Override @Override
public void hflush() throws IOException { public void hflush() throws IOException {
if (supportFlush) { if (supportFlush) {
flushInternal(); flushInternal(false);
} }
} }
@ -230,7 +230,7 @@ public synchronized void close() throws IOException {
} }
try { try {
flushInternal(); flushInternal(true);
threadExecutor.shutdown(); threadExecutor.shutdown();
} finally { } finally {
lastError = new IOException(FSExceptionMessages.STREAM_IS_CLOSED); lastError = new IOException(FSExceptionMessages.STREAM_IS_CLOSED);
@ -244,10 +244,10 @@ public synchronized void close() throws IOException {
} }
} }
private synchronized void flushInternal() throws IOException { private synchronized void flushInternal(boolean isClose) throws IOException {
maybeThrowLastError(); maybeThrowLastError();
writeCurrentBufferToService(); writeCurrentBufferToService();
flushWrittenBytesToService(); flushWrittenBytesToService(isClose);
} }
private synchronized void flushInternalAsync() throws IOException { private synchronized void flushInternalAsync() throws IOException {
@ -288,7 +288,7 @@ public Void call() throws Exception {
shrinkWriteOperationQueue(); shrinkWriteOperationQueue();
} }
private synchronized void flushWrittenBytesToService() throws IOException { private synchronized void flushWrittenBytesToService(boolean isClose) throws IOException {
for (WriteOperation writeOperation : writeOperations) { for (WriteOperation writeOperation : writeOperations) {
try { try {
writeOperation.task.get(); writeOperation.task.get();
@ -306,21 +306,22 @@ private synchronized void flushWrittenBytesToService() throws IOException {
throw lastError; throw lastError;
} }
} }
flushWrittenBytesToServiceInternal(position, false); flushWrittenBytesToServiceInternal(position, false, isClose);
} }
private synchronized void flushWrittenBytesToServiceAsync() throws IOException { private synchronized void flushWrittenBytesToServiceAsync() throws IOException {
shrinkWriteOperationQueue(); shrinkWriteOperationQueue();
if (this.lastTotalAppendOffset > this.lastFlushOffset) { if (this.lastTotalAppendOffset > this.lastFlushOffset) {
this.flushWrittenBytesToServiceInternal(this.lastTotalAppendOffset, true); this.flushWrittenBytesToServiceInternal(this.lastTotalAppendOffset, true,
false/*Async flush on close not permitted*/);
} }
} }
private synchronized void flushWrittenBytesToServiceInternal(final long offset, private synchronized void flushWrittenBytesToServiceInternal(final long offset,
final boolean retainUncommitedData) throws IOException { final boolean retainUncommitedData, final boolean isClose) throws IOException {
try { try {
client.flush(path, offset, retainUncommitedData); client.flush(path, offset, retainUncommitedData, isClose);
} catch (AzureBlobFileSystemException ex) { } catch (AzureBlobFileSystemException ex) {
if (ex instanceof AbfsRestOperationException) { if (ex instanceof AbfsRestOperationException) {
if (((AbfsRestOperationException) ex).getStatusCode() == HttpURLConnection.HTTP_NOT_FOUND) { if (((AbfsRestOperationException) ex).getStatusCode() == HttpURLConnection.HTTP_NOT_FOUND) {