big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-8742. Inotify: Support event for OP_TRUNCATE. Contributed by Surendra Singh Lilhore.

Browse Source
This commit is contained in:
Akira Ajisaka 2015-07-15 04:41:54 +09:00
parent 59388a8015
commit 979c9ca2ca
6 changed files with 83 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/inotify/Event.java
View File

@ -35,7 +35,7 @@
@InterfaceStability.Unstable
public abstract class Event {
public static enum EventType {
CREATE, CLOSE, APPEND, RENAME, METADATA, UNLINK
CREATE, CLOSE, APPEND, RENAME, METADATA, UNLINK, TRUNCATE
}
private EventType eventType;
@ -542,4 +542,39 @@ public long getTimestamp() {
return timestamp;
}
}
/**
* Sent when a file is truncated.
*/
public static class TruncateEvent extends Event {
private String path;
private long fileSize;
private long timestamp;
public TruncateEvent(String path, long fileSize, long timestamp) {
super(EventType.TRUNCATE);
this.path = path;
this.fileSize = fileSize;
this.timestamp = timestamp;
}
public String getPath() {
return path;
}
/**
* The size of the truncated file in bytes.
*/
public long getFileSize() {
return fileSize;
}
/**
* The time when this event occurred, in milliseconds since the epoch.
*/
public long getTimestamp() {
return timestamp;
}
}
}

7
hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/inotify.proto
View File

@ -41,6 +41,7 @@ enum EventType {
EVENT_RENAME = 0x3;
EVENT_METADATA = 0x4;
EVENT_UNLINK = 0x5;
EVENT_TRUNCATE = 0x6;
}
message EventProto {
@ -87,6 +88,12 @@ message CloseEventProto {
required int64 timestamp = 3;
}
message TruncateEventProto {
required string path = 1;
required int64 fileSize = 2;
required int64 timestamp = 3;
}
message AppendEventProto {
required string path = 1;
optional bool newBlock = 2 [default = false];

3
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
View File

@ -719,6 +719,9 @@ Release 2.8.0 - UNRELEASED
HDFS-8541. Mover should exit with NO_MOVE_PROGRESS if there is no move
progress. (Surendra Singh Lilhore via szetszwo)
HDFS-8742. Inotify: Support event for OP_TRUNCATE.
(Surendra Singh Lilhore via aajisaka)
OPTIMIZATIONS
HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than

17
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
View File

@ -2685,6 +2685,12 @@ public static EventBatchList convert(GetEditsFromTxidResponseProto resp) throws
.timestamp(unlink.getTimestamp())
.build());
break;
case EVENT_TRUNCATE:
InotifyProtos.TruncateEventProto truncate =
InotifyProtos.TruncateEventProto.parseFrom(p.getContents());
events.add(new Event.TruncateEvent(truncate.getPath(),
truncate.getFileSize(), truncate.getTimestamp()));
break;
default:
throw new RuntimeException("Unexpected inotify event type: " +
p.getType());
@ -2791,6 +2797,17 @@ public static GetEditsFromTxidResponseProto convertEditsResponse(EventBatchList
.setTimestamp(ue.getTimestamp()).build().toByteString()
).build());
break;
case TRUNCATE:
Event.TruncateEvent te = (Event.TruncateEvent) e;
events.add(InotifyProtos.EventProto.newBuilder()
.setType(InotifyProtos.EventType.EVENT_TRUNCATE)
.setContents(
InotifyProtos.TruncateEventProto.newBuilder()
.setPath(te.getPath())
.setFileSize(te.getFileSize())
.setTimestamp(te.getTimestamp()).build().toByteString()
).build());
break;
default:
throw new RuntimeException("Unexpected inotify event: " + e);
}

4
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/InotifyFSEditLogOpTranslator.java
View File

@ -176,6 +176,10 @@ public static EventBatch translate(FSEditLogOp op) {
.metadataType(Event.MetadataUpdateEvent.MetadataType.ACLS)
.path(saOp.src)
.acls(saOp.aclEntries).build() });
case OP_TRUNCATE:
FSEditLogOp.TruncateOp tOp = (FSEditLogOp.TruncateOp) op;
return new EventBatch(op.txid, new Event[] {
new Event.TruncateEvent(tOp.src, tOp.newLength, tOp.timestamp) });
default:
return null;
}

17
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSInotifyEventInputStream.java
View File

@ -102,6 +102,8 @@ public void testBasic() throws IOException, URISyntaxException,
DFSTestUtil.createFile(fs, new Path("/file"), BLOCK_SIZE, (short) 1, 0L);
DFSTestUtil.createFile(fs, new Path("/file3"), BLOCK_SIZE, (short) 1, 0L);
DFSTestUtil.createFile(fs, new Path("/file5"), BLOCK_SIZE, (short) 1, 0L);
DFSTestUtil.createFile(fs, new Path("/truncate_file"),
BLOCK_SIZE * 2, (short) 1, 0L);
DFSInotifyEventInputStream eis = client.getInotifyEventStream();
client.rename("/file", "/file4", null); // RenameOp -> RenameEvent
client.rename("/file4", "/file2"); // RenameOldOp -> RenameEvent
@ -136,7 +138,8 @@ public void testBasic() throws IOException, URISyntaxException,
"user::rwx,user:foo:rw-,group::r--,other::---", true));
client.removeAcl("/file5"); // SetAclOp -> MetadataUpdateEvent
client.rename("/file5", "/dir"); // RenameOldOp -> RenameEvent
//TruncateOp -> TruncateEvent
client.truncate("/truncate_file", BLOCK_SIZE);
EventBatch batch = null;
// RenameOp
@ -354,6 +357,18 @@ public void testBasic() throws IOException, URISyntaxException,
Assert.assertTrue(re3.getSrcPath().equals("/file5"));
Assert.assertTrue(re.getTimestamp() > 0);
// TruncateOp
batch = waitForNextEvents(eis);
Assert.assertEquals(1, batch.getEvents().length);
txid = checkTxid(batch, txid);
Assert
.assertTrue(batch.getEvents()[0].getEventType() ==
Event.EventType.TRUNCATE);
Event.TruncateEvent et = ((Event.TruncateEvent) batch.getEvents()[0]);
Assert.assertTrue(et.getPath().equals("/truncate_file"));
Assert.assertTrue(et.getFileSize() == BLOCK_SIZE);
Assert.assertTrue(et.getTimestamp() > 0);
// Returns null when there are no further events
Assert.assertTrue(eis.poll() == null);