diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 7b25112e2c..a14099e39a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -444,6 +444,9 @@ Release 2.0.0 - UNRELEASED HDFS-3369. Rename {get|set|add}INode(..) methods in BlockManager and BlocksMap to {get|set|add}BlockCollection(..). (John George via szetszwo) + HDFS-3134. harden edit log loader against malformed or malicious input. + (Colin Patrick McCabe via eli) + OPTIMIZATIONS HDFS-3024. Improve performance of stringification in addStoredBlock (todd) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/block/BlockTokenIdentifier.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/block/BlockTokenIdentifier.java index c1fd3f9f82..62f2d76237 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/block/BlockTokenIdentifier.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/block/BlockTokenIdentifier.java @@ -148,7 +148,8 @@ public void readFields(DataInput in) throws IOException { userId = WritableUtils.readString(in); blockPoolId = WritableUtils.readString(in); blockId = WritableUtils.readVLong(in); - int length = WritableUtils.readVInt(in); + int length = WritableUtils.readVIntInRange(in, 0, + AccessMode.class.getEnumConstants().length); for (int i = 0; i < length; i++) { modes.add(WritableUtils.readEnum(in, AccessMode.class)); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java index 56a610f101..9f7742cc67 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java @@ -203,6 +203,10 @@ T setBlockSize(long blockSize) { } T setBlocks(Block[] blocks) { + if (blocks.length > MAX_BLOCKS) { + throw new RuntimeException("Can't have more than " + MAX_BLOCKS + + " in an AddCloseOp."); + } this.blocks = blocks; return (T)this; } @@ -296,10 +300,18 @@ void readFields(DataInputStream in, int logVersion) } } + static final public int MAX_BLOCKS = 1024 * 1024 * 64; + private static Block[] readBlocks( DataInputStream in, int logVersion) throws IOException { int numBlocks = in.readInt(); + if (numBlocks < 0) { + throw new IOException("invalid negative number of blocks"); + } else if (numBlocks > MAX_BLOCKS) { + throw new IOException("invalid number of blocks: " + numBlocks + + ". The maximum number of blocks per file is " + MAX_BLOCKS); + } Block[] blocks = new Block[numBlocks]; for (int i = 0; i < numBlocks; i++) { Block blk = new Block(); @@ -579,6 +591,7 @@ static class ConcatDeleteOp extends FSEditLogOp { String trg; String[] srcs; long timestamp; + final static public int MAX_CONCAT_SRC = 1024 * 1024; private ConcatDeleteOp() { super(OP_CONCAT_DELETE); @@ -594,7 +607,12 @@ ConcatDeleteOp setTarget(String trg) { } ConcatDeleteOp setSources(String[] srcs) { + if (srcs.length > MAX_CONCAT_SRC) { + throw new RuntimeException("ConcatDeleteOp can only have " + + MAX_CONCAT_SRC + " sources at most."); + } this.srcs = srcs; + return this; } @@ -624,8 +642,8 @@ void readFields(DataInputStream in, int logVersion) if (!LayoutVersion.supports(Feature.EDITLOG_OP_OPTIMIZATION, logVersion)) { this.length = in.readInt(); if (length < 3) { // trg, srcs.., timestamp - throw new IOException("Incorrect data format. " - + "Concat delete operation."); + throw new IOException("Incorrect data format " + + "for ConcatDeleteOp."); } } this.trg = FSImageSerialization.readString(in); @@ -635,6 +653,15 @@ void readFields(DataInputStream in, int logVersion) } else { srcSize = this.length - 1 - 1; // trg and timestamp } + if (srcSize < 0) { + throw new IOException("Incorrect data format. " + + "ConcatDeleteOp cannot have a negative number of data " + + " sources."); + } else if (srcSize > MAX_CONCAT_SRC) { + throw new IOException("Incorrect data format. " + + "ConcatDeleteOp can have at most " + MAX_CONCAT_SRC + + " sources, but we tried to have " + (length - 3) + " sources."); + } this.srcs = new String [srcSize]; for(int i=0; i