From 82ef9accafe7318278efb169678e17065e082c8e Mon Sep 17 00:00:00 2001 From: Rakesh Radhakrishnan Date: Wed, 1 Mar 2017 16:06:02 +0530 Subject: [PATCH] HDFS-11428. Change setErasureCodingPolicy to take a required string EC policy name. Contributed by Andrew Wang. --- .../org/apache/hadoop/hdfs/DFSClient.java | 4 +- .../hadoop/hdfs/DistributedFileSystem.java | 11 +++--- .../apache/hadoop/hdfs/client/HdfsAdmin.java | 10 +++-- .../hadoop/hdfs/protocol/ClientProtocol.java | 5 +-- .../ClientNamenodeProtocolTranslatorPB.java | 6 +-- .../src/main/proto/erasurecoding.proto | 2 +- ...amenodeProtocolServerSideTranslatorPB.java | 4 +- .../server/namenode/FSDirErasureCodingOp.java | 13 ++++++- .../hdfs/server/namenode/FSNamesystem.java | 9 +++-- .../server/namenode/NameNodeRpcServer.java | 4 +- .../org/apache/hadoop/hdfs/tools/ECAdmin.java | 26 +------------ .../org/apache/hadoop/hdfs/DFSTestUtil.java | 6 ++- .../hdfs/ErasureCodeBenchmarkThroughput.java | 3 +- .../hdfs/TestDFSStripedInputStream.java | 3 +- .../hdfs/TestDFSStripedOutputStream.java | 3 +- ...TestDFSStripedOutputStreamWithFailure.java | 2 +- .../hdfs/TestDecommissionWithStriped.java | 3 +- .../hdfs/TestErasureCodingPolicies.java | 37 +++++++++++-------- .../TestErasureCodingPolicyWithSnapshot.java | 10 ++--- .../apache/hadoop/hdfs/TestFileChecksum.java | 3 +- .../hdfs/TestFileStatusWithECPolicy.java | 2 +- .../hadoop/hdfs/TestLeaseRecoveryStriped.java | 2 +- .../hdfs/TestReadStripedFileWithDecoding.java | 3 +- .../TestReadStripedFileWithMissingBlocks.java | 2 +- .../hdfs/TestReconstructStripedFile.java | 3 +- .../hdfs/TestSafeModeWithStripedFile.java | 3 +- .../TestUnsetAndChangeDirectoryEcPolicy.java | 18 ++++----- .../hadoop/hdfs/TestWriteReadStripedFile.java | 3 +- .../hdfs/TestWriteStripedFileWithFailure.java | 3 +- .../hdfs/server/balancer/TestBalancer.java | 3 +- .../TestBlockTokenWithDFSStriped.java | 4 +- ...nstructStripedBlocksWithRackAwareness.java | 9 +++-- .../TestSequentialBlockGroupId.java | 4 +- .../TestDataNodeErasureCodingMetrics.java | 3 +- .../hadoop/hdfs/server/mover/TestMover.java | 3 +- .../TestAddOverReplicatedStripedBlocks.java | 3 +- .../namenode/TestAddStripedBlockInFBR.java | 3 +- .../server/namenode/TestAddStripedBlocks.java | 3 +- .../server/namenode/TestFSEditLogLoader.java | 4 +- .../hdfs/server/namenode/TestFSImage.java | 6 +-- .../hadoop/hdfs/server/namenode/TestFsck.java | 8 ++-- .../server/namenode/TestNameNodeMXBean.java | 3 +- .../namenode/TestQuotaWithStripedBlocks.java | 3 +- .../TestReconstructStripedBlocks.java | 6 ++- .../server/namenode/TestStripedINodeFile.java | 6 ++- .../TestOfflineImageViewer.java | 3 +- ...stOfflineImageViewerWithStripedBlocks.java | 3 +- .../test/resources/testErasureCodingConf.xml | 2 +- .../java/org/apache/hadoop/fs/TestDFSIO.java | 2 +- 49 files changed, 152 insertions(+), 132 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java index e0ccd620bd..ae1d8217b9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java @@ -2611,12 +2611,12 @@ public RemoteIterator listEncryptionZones() } - public void setErasureCodingPolicy(String src, ErasureCodingPolicy ecPolicy) + public void setErasureCodingPolicy(String src, String ecPolicyName) throws IOException { checkOpen(); try (TraceScope ignored = newPathTraceScope("setErasureCodingPolicy", src)) { - namenode.setErasureCodingPolicy(src, ecPolicy); + namenode.setErasureCodingPolicy(src, ecPolicyName); } catch (RemoteException re) { throw re.unwrapRemoteException(AccessControlException.class, SafeModeException.class, diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java index 30dcfa4e82..1eef560ec5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java @@ -2397,17 +2397,18 @@ public DFSInotifyEventInputStream getInotifyEventStream(long lastReadTxid) * Set the source path to the specified erasure coding policy. * * @param path The directory to set the policy - * @param ecPolicy The erasure coding policy. If not specified default will - * be used. + * @param ecPolicyName The erasure coding policy name. * @throws IOException */ public void setErasureCodingPolicy(final Path path, - final ErasureCodingPolicy ecPolicy) throws IOException { + final String ecPolicyName) throws IOException { Path absF = fixRelativePart(path); + Preconditions.checkNotNull(ecPolicyName, "Erasure coding policy cannot be" + + " null."); new FileSystemLinkResolver() { @Override public Void doCall(final Path p) throws IOException { - dfs.setErasureCodingPolicy(getPathName(p), ecPolicy); + dfs.setErasureCodingPolicy(getPathName(p), ecPolicyName); return null; } @@ -2415,7 +2416,7 @@ public Void doCall(final Path p) throws IOException { public Void next(final FileSystem fs, final Path p) throws IOException { if (fs instanceof DistributedFileSystem) { DistributedFileSystem myDfs = (DistributedFileSystem) fs; - myDfs.setErasureCodingPolicy(p, ecPolicy); + myDfs.setErasureCodingPolicy(p, ecPolicyName); return null; } throw new UnsupportedOperationException( diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java index a192fa87eb..456f280f38 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java @@ -465,13 +465,15 @@ public Collection getAllStoragePolicies() * Set the source path to the specified erasure coding policy. * * @param path The source path referring to a directory. - * @param ecPolicy The erasure coding policy for the directory. - * If null, the default will be used. + * @param ecPolicyName The erasure coding policy name for the directory. + * * @throws IOException + * @throws HadoopIllegalArgumentException if the specified EC policy is not + * enabled on the cluster */ public void setErasureCodingPolicy(final Path path, - final ErasureCodingPolicy ecPolicy) throws IOException { - dfs.setErasureCodingPolicy(path, ecPolicy); + final String ecPolicyName) throws IOException { + dfs.setErasureCodingPolicy(path, ecPolicyName); } /** diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java index eaebd6fdf7..041d226840 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java @@ -1510,11 +1510,10 @@ List listXAttrs(String src) /** * Set an erasure coding policy on a specified path. * @param src The path to set policy on. - * @param ecPolicy The erasure coding policy. If null, default policy will - * be used + * @param ecPolicyName The erasure coding policy name. */ @AtMostOnce - void setErasureCodingPolicy(String src, ErasureCodingPolicy ecPolicy) + void setErasureCodingPolicy(String src, String ecPolicyName) throws IOException; /** diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java index de474b59fb..da4a17f579 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java @@ -1459,14 +1459,12 @@ public BatchedEntries listEncryptionZones(long id) } @Override - public void setErasureCodingPolicy(String src, ErasureCodingPolicy ecPolicy) + public void setErasureCodingPolicy(String src, String ecPolicyName) throws IOException { final SetErasureCodingPolicyRequestProto.Builder builder = SetErasureCodingPolicyRequestProto.newBuilder(); builder.setSrc(src); - if (ecPolicy != null) { - builder.setEcPolicy(PBHelperClient.convertErasureCodingPolicy(ecPolicy)); - } + builder.setEcPolicyName(ecPolicyName); SetErasureCodingPolicyRequestProto req = builder.build(); try { rpcProxy.setErasureCodingPolicy(null, req); diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/erasurecoding.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/erasurecoding.proto index 7ea8d32614..4eab4d34ad 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/erasurecoding.proto +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/erasurecoding.proto @@ -25,7 +25,7 @@ import "hdfs.proto"; message SetErasureCodingPolicyRequestProto { required string src = 1; - optional ErasureCodingPolicyProto ecPolicy = 2; + required string ecPolicyName = 2; } message SetErasureCodingPolicyResponseProto { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java index 1aa15c8b6f..1944fe7a28 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java @@ -1442,9 +1442,7 @@ public SetErasureCodingPolicyResponseProto setErasureCodingPolicy( RpcController controller, SetErasureCodingPolicyRequestProto req) throws ServiceException { try { - ErasureCodingPolicy ecPolicy = req.hasEcPolicy() ? - PBHelperClient.convertErasureCodingPolicy(req.getEcPolicy()) : null; - server.setErasureCodingPolicy(req.getSrc(), ecPolicy); + server.setErasureCodingPolicy(req.getSrc(), req.getEcPolicyName()); return SetErasureCodingPolicyResponseProto.newBuilder().build(); } catch (IOException e) { throw new ServiceException(e); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java index 0ab8c89d79..f4c4ad7a01 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java @@ -59,14 +59,16 @@ private FSDirErasureCodingOp() {} * * @param fsn The namespace * @param srcArg The path of the target directory. - * @param ecPolicy The erasure coding policy to set on the target directory. + * @param ecPolicyName The erasure coding policy name to set on the target + * directory. * @param logRetryCache whether to record RPC ids in editlog for retry * cache rebuilding * @return {@link HdfsFileStatus} * @throws IOException + * @throws HadoopIllegalArgumentException if the policy is not enabled */ static HdfsFileStatus setErasureCodingPolicy(final FSNamesystem fsn, - final String srcArg, final ErasureCodingPolicy ecPolicy, + final String srcArg, final String ecPolicyName, final boolean logRetryCache) throws IOException { assert fsn.hasWriteLock(); @@ -78,6 +80,13 @@ static HdfsFileStatus setErasureCodingPolicy(final FSNamesystem fsn, List xAttrs; fsd.writeLock(); try { + ErasureCodingPolicy ecPolicy = fsn.getErasureCodingPolicyManager() + .getPolicyByName(ecPolicyName); + if (ecPolicy == null) { + throw new HadoopIllegalArgumentException("Policy '" + + ecPolicyName + "' does not match any supported erasure coding " + + "policies."); + } iip = fsd.resolvePath(pc, src, DirOp.WRITE_LINK); src = iip.getPath(); xAttrs = setErasureCodingPolicyXAttr(fsn, iip, ecPolicy); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index 38a326c7dd..254ccfa492 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -6772,13 +6772,14 @@ BatchedListEntries listEncryptionZones(long prevId) /** * Set an erasure coding policy on the given path. * @param srcArg The path of the target directory. - * @param ecPolicy The erasure coding policy to set on the target directory. + * @param ecPolicyName The erasure coding policy to set on the target + * directory. * @throws AccessControlException if the caller is not the superuser. * @throws UnresolvedLinkException if the path can't be resolved. * @throws SafeModeException if the Namenode is in safe mode. */ - void setErasureCodingPolicy(final String srcArg, final ErasureCodingPolicy - ecPolicy, final boolean logRetryCache) throws IOException, + void setErasureCodingPolicy(final String srcArg, final String ecPolicyName, + final boolean logRetryCache) throws IOException, UnresolvedLinkException, SafeModeException, AccessControlException { final String operationName = "setErasureCodingPolicy"; checkSuperuserPrivilege(); @@ -6790,7 +6791,7 @@ void setErasureCodingPolicy(final String srcArg, final ErasureCodingPolicy checkOperation(OperationCategory.WRITE); checkNameNodeSafeMode("Cannot set erasure coding policy on " + srcArg); resultingStat = FSDirErasureCodingOp.setErasureCodingPolicy(this, - srcArg, ecPolicy, logRetryCache); + srcArg, ecPolicyName, logRetryCache); success = true; } catch (AccessControlException ace) { logAuditEvent(success, operationName, srcArg, null, diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java index f9cfa42f3a..cf14e8afd7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java @@ -2018,7 +2018,7 @@ public BatchedEntries listEncryptionZones( } @Override // ClientProtocol - public void setErasureCodingPolicy(String src, ErasureCodingPolicy ecPolicy) + public void setErasureCodingPolicy(String src, String ecPolicyName) throws IOException { checkNNStartup(); final CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache); @@ -2027,7 +2027,7 @@ public void setErasureCodingPolicy(String src, ErasureCodingPolicy ecPolicy) } boolean success = false; try { - namesystem.setErasureCodingPolicy(src, ecPolicy, cacheEntry != null); + namesystem.setErasureCodingPolicy(src, ecPolicyName, cacheEntry != null); success = true; } finally { RetryCache.setState(cacheEntry, success); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java index 29c65b10fc..548f754a1e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java @@ -28,7 +28,6 @@ import org.apache.hadoop.util.ToolRunner; import java.io.IOException; -import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.LinkedList; @@ -227,30 +226,7 @@ public int run(Configuration conf, List args) throws IOException { final Path p = new Path(path); final DistributedFileSystem dfs = AdminHelper.getDFS(p.toUri(), conf); try { - ErasureCodingPolicy ecPolicy = null; - ErasureCodingPolicy[] ecPolicies = - dfs.getClient().getErasureCodingPolicies(); - for (ErasureCodingPolicy policy : ecPolicies) { - if (ecPolicyName.equals(policy.getName())) { - ecPolicy = policy; - break; - } - } - if (ecPolicy == null) { - StringBuilder sb = new StringBuilder(); - sb.append("Policy '"); - sb.append(ecPolicyName); - sb.append("' does not match any of the supported policies."); - sb.append(" Please select any one of "); - List ecPolicyNames = new ArrayList(); - for (ErasureCodingPolicy policy : ecPolicies) { - ecPolicyNames.add(policy.getName()); - } - sb.append(ecPolicyNames); - System.err.println(sb.toString()); - return 3; - } - dfs.setErasureCodingPolicy(p, ecPolicy); + dfs.setErasureCodingPolicy(p, ecPolicyName); System.out.println("Set erasure coding policy " + ecPolicyName + " on " + path); } catch (Exception e) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java index 1fbc1d9207..60a1a4e437 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java @@ -136,6 +136,7 @@ import org.apache.hadoop.hdfs.server.datanode.SimulatedFSDataset; import org.apache.hadoop.hdfs.server.datanode.TestTransferRbw; import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi; +import org.apache.hadoop.hdfs.server.namenode.ErasureCodingPolicyManager; import org.apache.hadoop.hdfs.server.namenode.FSDirectory; import org.apache.hadoop.hdfs.server.namenode.FSEditLog; import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; @@ -1898,7 +1899,7 @@ public static void createStripedFile(MiniDFSCluster cluster, Path file, Path dir, int numBlocks, int numStripesPerBlk, boolean toMkdir) throws Exception { createStripedFile(cluster, file, dir, numBlocks, numStripesPerBlk, - toMkdir, null); + toMkdir, ErasureCodingPolicyManager.getSystemDefaultPolicy()); } /** @@ -1922,7 +1923,8 @@ public static void createStripedFile(MiniDFSCluster cluster, Path file, assert dir != null; dfs.mkdirs(dir); try { - dfs.getClient().setErasureCodingPolicy(dir.toString(), ecPolicy); + dfs.getClient() + .setErasureCodingPolicy(dir.toString(), ecPolicy.getName()); } catch (IOException e) { if (!e.getMessage().contains("non-empty directory")) { throw e; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/ErasureCodeBenchmarkThroughput.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/ErasureCodeBenchmarkThroughput.java index 13dc997432..d1a7569fa0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/ErasureCodeBenchmarkThroughput.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/ErasureCodeBenchmarkThroughput.java @@ -192,7 +192,8 @@ private void setUpDir() throws IOException { } if (!dfs.exists(ecPath)) { dfs.mkdirs(ecPath); - dfs.getClient().setErasureCodingPolicy(ecPath.toString(), ecPolicy); + dfs.getClient() + .setErasureCodingPolicy(ecPath.toString(), ecPolicy.getName()); } else { Preconditions.checkArgument( dfs.getClient(). diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java index 9b699045b5..50b366b26a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java @@ -108,7 +108,8 @@ public void setup() throws IOException { } fs = cluster.getFileSystem(); fs.mkdirs(dirPath); - fs.getClient().setErasureCodingPolicy(dirPath.toString(), ecPolicy); + fs.getClient() + .setErasureCodingPolicy(dirPath.toString(), ecPolicy.getName()); } @After diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java index 8834e14c22..5956cc1239 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java @@ -89,7 +89,8 @@ public void setup() throws IOException { NativeRSRawErasureCoderFactory.class.getCanonicalName()); } cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build(); - cluster.getFileSystem().getClient().setErasureCodingPolicy("/", ecPolicy); + cluster.getFileSystem().getClient().setErasureCodingPolicy("/", ecPolicy + .getName()); fs = cluster.getFileSystem(); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java index fabd4176da..01bd3c569e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java @@ -221,7 +221,7 @@ private void setup(Configuration conf) throws IOException { cluster.waitActive(); dfs = cluster.getFileSystem(); dfs.mkdirs(dir); - dfs.setErasureCodingPolicy(dir, ecPolicy); + dfs.setErasureCodingPolicy(dir, ecPolicy.getName()); } private void tearDown() { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommissionWithStriped.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommissionWithStriped.java index 2eab3092b9..f8ed0c3f69 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommissionWithStriped.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommissionWithStriped.java @@ -142,7 +142,8 @@ public void setup() throws IOException { client = getDfsClient(cluster.getNameNode(0), conf); dfs.mkdirs(ecDir); - dfs.setErasureCodingPolicy(ecDir, null); + dfs.setErasureCodingPolicy(ecDir, + ErasureCodingPolicyManager.getSystemDefaultPolicy().getName()); } @After diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java index 0e431fae27..8608626e59 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java @@ -47,6 +47,8 @@ public class TestErasureCodingPolicies { private MiniDFSCluster cluster; private DistributedFileSystem fs; private static final int BLOCK_SIZE = 1024; + private static final ErasureCodingPolicy EC_POLICY = + ErasureCodingPolicyManager.getSystemDefaultPolicy(); private FSNamesystem namesystem; @Before @@ -80,7 +82,8 @@ public void testReplicatedFileUnderECDir() throws IOException { DFSTestUtil.createFile(fs, replicatedFile, 0, (short) 3, 0L); // set ec policy on dir - fs.setErasureCodingPolicy(dir, null); + fs.setErasureCodingPolicy(dir, + ErasureCodingPolicyManager.getSystemDefaultPolicy().getName()); // create a file which should be using ec final Path ecSubDir = new Path(dir, "ecSubDir"); final Path ecFile = new Path(ecSubDir, "ecFile"); @@ -132,7 +135,7 @@ public void testBasicSetECPolicy() fs.mkdir(testDir, FsPermission.getDirDefault()); /* Normal creation of an erasure coding directory */ - fs.getClient().setErasureCodingPolicy(testDir.toString(), null); + fs.setErasureCodingPolicy(testDir, EC_POLICY.getName()); /* Verify files under the directory are striped */ final Path ECFilePath = new Path(testDir, "foo"); @@ -148,7 +151,7 @@ public void testBasicSetECPolicy() fs.mkdir(notEmpty, FsPermission.getDirDefault()); final Path oldFile = new Path(notEmpty, "old"); fs.create(oldFile); - fs.getClient().setErasureCodingPolicy(notEmpty.toString(), null); + fs.setErasureCodingPolicy(notEmpty, EC_POLICY.getName()); final Path newFile = new Path(notEmpty, "new"); fs.create(newFile); INode oldInode = namesystem.getFSDirectory().getINode(oldFile.toString()); @@ -160,10 +163,10 @@ public void testBasicSetECPolicy() final Path dir1 = new Path("/dir1"); final Path dir2 = new Path(dir1, "dir2"); fs.mkdir(dir1, FsPermission.getDirDefault()); - fs.getClient().setErasureCodingPolicy(dir1.toString(), null); + fs.setErasureCodingPolicy(dir1, EC_POLICY.getName()); fs.mkdir(dir2, FsPermission.getDirDefault()); try { - fs.getClient().setErasureCodingPolicy(dir2.toString(), null); + fs.setErasureCodingPolicy(dir2, EC_POLICY.getName()); } catch (IOException e) { fail("Nested erasure coding policies are supported"); } @@ -172,7 +175,7 @@ public void testBasicSetECPolicy() final Path fPath = new Path("/file"); fs.create(fPath); try { - fs.getClient().setErasureCodingPolicy(fPath.toString(), null); + fs.setErasureCodingPolicy(fPath, EC_POLICY.getName()); fail("Erasure coding policy on file"); } catch (IOException e) { assertExceptionContains("erasure coding policy for a file", e); @@ -185,8 +188,8 @@ public void testMoveValidity() throws IOException, InterruptedException { final Path dstECDir = new Path("/dstEC"); fs.mkdir(srcECDir, FsPermission.getDirDefault()); fs.mkdir(dstECDir, FsPermission.getDirDefault()); - fs.getClient().setErasureCodingPolicy(srcECDir.toString(), null); - fs.getClient().setErasureCodingPolicy(dstECDir.toString(), null); + fs.setErasureCodingPolicy(srcECDir, EC_POLICY.getName()); + fs.setErasureCodingPolicy(dstECDir, EC_POLICY.getName()); final Path srcFile = new Path(srcECDir, "foo"); fs.create(srcFile); @@ -220,7 +223,8 @@ public void testMoveValidity() throws IOException, InterruptedException { public void testReplication() throws IOException { final Path testDir = new Path("/ec"); fs.mkdir(testDir, FsPermission.getDirDefault()); - fs.setErasureCodingPolicy(testDir, null); + fs.setErasureCodingPolicy(testDir, + ErasureCodingPolicyManager.getSystemDefaultPolicy().getName()); final Path fooFile = new Path(testDir, "foo"); // create ec file with replication=0 fs.create(fooFile, FsPermission.getFileDefault(), true, @@ -241,8 +245,9 @@ public void testGetErasureCodingPolicyWithSystemDefaultECPolicy() throws Excepti // dir EC policy should be null assertNull(fs.getClient().getFileInfo(src).getErasureCodingPolicy()); // dir EC policy after setting - fs.getClient().setErasureCodingPolicy(src, null); //Default one will be used. - ErasureCodingPolicy sysDefaultECPolicy = ErasureCodingPolicyManager.getSystemDefaultPolicy(); + ErasureCodingPolicy sysDefaultECPolicy = + ErasureCodingPolicyManager.getSystemDefaultPolicy(); + fs.getClient().setErasureCodingPolicy(src, sysDefaultECPolicy.getName()); verifyErasureCodingInfo(src, sysDefaultECPolicy); fs.create(new Path(ecDir, "child1")).close(); // verify for the files in ec dir @@ -263,7 +268,7 @@ public void testGetErasureCodingPolicy() throws Exception { // dir ECInfo before being set assertNull(fs.getClient().getFileInfo(src).getErasureCodingPolicy()); // dir ECInfo after set - fs.getClient().setErasureCodingPolicy(src, usingECPolicy); + fs.getClient().setErasureCodingPolicy(src, usingECPolicy.getName()); verifyErasureCodingInfo(src, usingECPolicy); fs.create(new Path(ecDir, "child1")).close(); // verify for the files in ec dir @@ -291,12 +296,12 @@ public void testCreationErasureCodingZoneWithInvalidPolicy() final Path ecDir = new Path(src); try { fs.mkdir(ecDir, FsPermission.getDirDefault()); - fs.getClient().setErasureCodingPolicy(src, ecPolicy); + fs.getClient().setErasureCodingPolicy(src, ecPolicy.getName()); fail("HadoopIllegalArgumentException should be thrown for" + "setting an invalid erasure coding policy"); } catch (Exception e) { - assertExceptionContains("Policy [ RS-4-2-128k ] does not match " + - "any of the supported policies",e); + assertExceptionContains("Policy 'RS-4-2-128k' does not match " + + "any supported erasure coding policies",e); } } @@ -338,7 +343,7 @@ public void testMultiplePoliciesCoExist() throws Exception { for (ErasureCodingPolicy policy : sysPolicies) { Path dir = new Path("/policy_" + policy.getId()); fs.mkdir(dir, FsPermission.getDefault()); - fs.setErasureCodingPolicy(dir, policy); + fs.setErasureCodingPolicy(dir, policy.getName()); Path file = new Path(dir, "child"); fs.create(file).close(); assertEquals(policy, fs.getErasureCodingPolicy(file)); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicyWithSnapshot.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicyWithSnapshot.java index e02c0bf3b0..6cf4ef42c7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicyWithSnapshot.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicyWithSnapshot.java @@ -75,7 +75,7 @@ public void testSnapshotsOnErasureCodingDirsParentDir() throws Exception { fs.mkdirs(ecDir); fs.allowSnapshot(ecDirParent); // set erasure coding policy - fs.setErasureCodingPolicy(ecDir, sysDefaultPolicy); + fs.setErasureCodingPolicy(ecDir, sysDefaultPolicy.getName()); DFSTestUtil.createFile(fs, ecFile, len, (short) 1, 0xFEED); String contents = DFSTestUtil.readFile(fs, ecFile); final Path snap1 = fs.createSnapshot(ecDirParent, "snap1"); @@ -93,7 +93,7 @@ public void testSnapshotsOnErasureCodingDirsParentDir() throws Exception { fs.getErasureCodingPolicy(snap2ECDir)); // Make dir again with system default ec policy - fs.setErasureCodingPolicy(ecDir, sysDefaultPolicy); + fs.setErasureCodingPolicy(ecDir, sysDefaultPolicy.getName()); final Path snap3 = fs.createSnapshot(ecDirParent, "snap3"); final Path snap3ECDir = new Path(snap3, ecDir.getName()); // Check that snap3's ECPolicy has the correct settings @@ -134,7 +134,7 @@ public void testSnapshotsOnErasureCodingDir() throws Exception { fs.mkdirs(ecDir); fs.allowSnapshot(ecDir); - fs.setErasureCodingPolicy(ecDir, sysDefaultPolicy); + fs.setErasureCodingPolicy(ecDir, sysDefaultPolicy.getName()); final Path snap1 = fs.createSnapshot(ecDir, "snap1"); assertEquals("Got unexpected erasure coding policy", sysDefaultPolicy, fs.getErasureCodingPolicy(snap1)); @@ -150,7 +150,7 @@ public void testSnapshotsOnErasureCodingDirAfterNNRestart() throws Exception { fs.allowSnapshot(ecDir); // set erasure coding policy - fs.setErasureCodingPolicy(ecDir, sysDefaultPolicy); + fs.setErasureCodingPolicy(ecDir, sysDefaultPolicy.getName()); final Path snap1 = fs.createSnapshot(ecDir, "snap1"); ErasureCodingPolicy ecSnap = fs.getErasureCodingPolicy(snap1); assertEquals("Got unexpected erasure coding policy", sysDefaultPolicy, @@ -182,7 +182,7 @@ public void testCopySnapshotWillNotPreserveErasureCodingPolicy() fs.allowSnapshot(ecDir); // set erasure coding policy - fs.setErasureCodingPolicy(ecDir, sysDefaultPolicy); + fs.setErasureCodingPolicy(ecDir, sysDefaultPolicy.getName()); DFSTestUtil.createFile(fs, ecFile, len, (short) 1, 0xFEED); final Path snap1 = fs.createSnapshot(ecDir, "snap1"); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileChecksum.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileChecksum.java index 9d60798784..8ae176ffef 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileChecksum.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileChecksum.java @@ -81,7 +81,8 @@ public void setup() throws IOException { cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build(); Path ecPath = new Path(ecDir); cluster.getFileSystem().mkdir(ecPath, FsPermission.getDirDefault()); - cluster.getFileSystem().getClient().setErasureCodingPolicy(ecDir, null); + cluster.getFileSystem().getClient().setErasureCodingPolicy(ecDir, + ErasureCodingPolicyManager.getSystemDefaultPolicy().getName()); fs = cluster.getFileSystem(); client = fs.getClient(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatusWithECPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatusWithECPolicy.java index 2210fbe2f4..d7b73274a5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatusWithECPolicy.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatusWithECPolicy.java @@ -73,7 +73,7 @@ public void testFileStatusWithECPolicy() throws Exception { final ErasureCodingPolicy ecPolicy1 = ErasureCodingPolicyManager.getSystemDefaultPolicy(); // set EC policy on dir - fs.setErasureCodingPolicy(dir, ecPolicy1); + fs.setErasureCodingPolicy(dir, ecPolicy1.getName()); final ErasureCodingPolicy ecPolicy2 = client.getFileInfo(dir.toUri().getPath()).getErasureCodingPolicy(); assertNotNull(ecPolicy2); assertTrue(ecPolicy1.equals(ecPolicy2)); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecoveryStriped.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecoveryStriped.java index 246818bdbc..710ff622b4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecoveryStriped.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecoveryStriped.java @@ -94,7 +94,7 @@ public void setup() throws IOException { cluster.waitActive(); dfs = cluster.getFileSystem(); dfs.mkdirs(dir); - dfs.setErasureCodingPolicy(dir, ecPolicy); + dfs.setErasureCodingPolicy(dir, ecPolicy.getName()); } @After diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReadStripedFileWithDecoding.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReadStripedFileWithDecoding.java index 506a005b36..d7d9cc1c20 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReadStripedFileWithDecoding.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReadStripedFileWithDecoding.java @@ -102,7 +102,8 @@ public void setup() throws IOException { conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_KEY, false); cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build(); - cluster.getFileSystem().getClient().setErasureCodingPolicy("/", null); + cluster.getFileSystem().getClient().setErasureCodingPolicy("/", + ErasureCodingPolicyManager.getSystemDefaultPolicy().getName()); fs = cluster.getFileSystem(); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReadStripedFileWithMissingBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReadStripedFileWithMissingBlocks.java index 2ee86639fe..b65626e149 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReadStripedFileWithMissingBlocks.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReadStripedFileWithMissingBlocks.java @@ -61,7 +61,7 @@ public void setup() throws IOException { conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_MAX_STREAMS_KEY, 0); cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build(); cluster.getFileSystem().getClient().setErasureCodingPolicy( - "/", ecPolicy); + "/", ecPolicy.getName()); fs = cluster.getFileSystem(); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReconstructStripedFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReconstructStripedFile.java index 08b53c2327..38939f5482 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReconstructStripedFile.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReconstructStripedFile.java @@ -107,7 +107,8 @@ public void setup() throws IOException { cluster.waitActive(); fs = cluster.getFileSystem(); - fs.getClient().setErasureCodingPolicy("/", null); + fs.getClient().setErasureCodingPolicy("/", + ErasureCodingPolicyManager.getSystemDefaultPolicy().getName()); List datanodes = cluster.getDataNodes(); for (int i = 0; i < dnNum; i++) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeModeWithStripedFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeModeWithStripedFile.java index 2d37c0673f..0731779681 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeModeWithStripedFile.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeModeWithStripedFile.java @@ -63,7 +63,8 @@ public void setup() throws IOException { conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize); conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, 100); cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build(); - cluster.getFileSystem().getClient().setErasureCodingPolicy("/", null); + cluster.getFileSystem().getClient().setErasureCodingPolicy("/", + ErasureCodingPolicyManager.getSystemDefaultPolicy().getName()); cluster.waitActive(); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestUnsetAndChangeDirectoryEcPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestUnsetAndChangeDirectoryEcPolicy.java index f26aa06a7c..86624cc05b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestUnsetAndChangeDirectoryEcPolicy.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestUnsetAndChangeDirectoryEcPolicy.java @@ -101,7 +101,7 @@ public void testUnsetEcPolicy() throws Exception { // Test unset a directory which has no EC policy fs.unsetErasureCodingPolicy(dirPath); // Set EC policy on directory - fs.setErasureCodingPolicy(dirPath, ecPolicy); + fs.setErasureCodingPolicy(dirPath, ecPolicy.getName()); DFSTestUtil.createFile(fs, ecFilePath, fileLen, (short) 1, 0L); fs.unsetErasureCodingPolicy(dirPath); @@ -142,12 +142,12 @@ public void testNestedEcPolicy() throws Exception { .getPolicyByPolicyID(HdfsConstants.RS_3_2_POLICY_ID); fs.mkdirs(parentDir); - fs.setErasureCodingPolicy(parentDir, ecPolicy); + fs.setErasureCodingPolicy(parentDir, ecPolicy.getName()); fs.mkdirs(childDir); // Create RS(6,3) EC policy file DFSTestUtil.createFile(fs, ec63FilePath, fileLen, (short) 1, 0L); // Set RS(3,2) EC policy on child directory - fs.setErasureCodingPolicy(childDir, ec32Policy); + fs.setErasureCodingPolicy(childDir, ec32Policy.getName()); // Create RS(3,2) EC policy file DFSTestUtil.createFile(fs, ec32FilePath, fileLen, (short) 1, 0L); @@ -202,7 +202,7 @@ public void testUnsetRootDirEcPolicy() throws Exception { // Test unset root path which has no EC policy fs.unsetErasureCodingPolicy(rootPath); // Set EC policy on root path - fs.setErasureCodingPolicy(rootPath, ecPolicy); + fs.setErasureCodingPolicy(rootPath, ecPolicy.getName()); DFSTestUtil.createFile(fs, ecFilePath, fileLen, (short) 1, 0L); fs.unsetErasureCodingPolicy(rootPath); DFSTestUtil.createFile(fs, replicateFilePath, fileLen, (short) 1, 0L); @@ -240,11 +240,11 @@ public void testChangeRootDirEcPolicy() throws Exception { .getPolicyByPolicyID(HdfsConstants.RS_3_2_POLICY_ID); fs.unsetErasureCodingPolicy(rootPath); - fs.setErasureCodingPolicy(rootPath, ecPolicy); + fs.setErasureCodingPolicy(rootPath, ecPolicy.getName()); // Create RS(6,3) EC policy file DFSTestUtil.createFile(fs, ec63FilePath, fileLen, (short) 1, 0L); // Change EC policy from RS(6,3) to RS(3,2) - fs.setErasureCodingPolicy(rootPath, ec32Policy); + fs.setErasureCodingPolicy(rootPath, ec32Policy.getName()); DFSTestUtil.createFile(fs, ec32FilePath, fileLen, (short) 1, 0L); // start to check @@ -281,7 +281,7 @@ public void testDifferentReplicaFactor() throws Exception { final Path replicateFilePath2 = new Path(ecDirPath, "rep_file2"); fs.mkdirs(ecDirPath); - fs.setErasureCodingPolicy(ecDirPath, ecPolicy); + fs.setErasureCodingPolicy(ecDirPath, ecPolicy.getName()); DFSTestUtil.createFile(fs, ecFilePath, fileLen, (short) 1, 0L); fs.unsetErasureCodingPolicy(ecDirPath); DFSTestUtil.createFile(fs, replicateFilePath, fileLen, (short) 3, 0L); @@ -328,7 +328,7 @@ public void testNonExistentDir() throws Exception { // Set EC policy on non-existent directory try { - fs.setErasureCodingPolicy(dirPath, ecPolicy); + fs.setErasureCodingPolicy(dirPath, ecPolicy.getName()); fail("FileNotFoundException should be thrown for a non-existent" + " file path"); } catch (FileNotFoundException e) { @@ -347,7 +347,7 @@ public void testEcPolicyOnFile() throws Exception { // Set EC policy on file try { - fs.setErasureCodingPolicy(ecFilePath, ecPolicy); + fs.setErasureCodingPolicy(ecFilePath, ecPolicy.getName()); fail("IOException should be thrown for setting EC policy on file"); } catch (IOException e) { assertExceptionContains("Attempt to set an erasure coding policy " + diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteReadStripedFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteReadStripedFile.java index 712f326856..76ca704e93 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteReadStripedFile.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteReadStripedFile.java @@ -80,7 +80,8 @@ public void setup() throws IOException { cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build(); fs = cluster.getFileSystem(); fs.mkdirs(new Path("/ec")); - cluster.getFileSystem().getClient().setErasureCodingPolicy("/ec", null); + cluster.getFileSystem().getClient().setErasureCodingPolicy("/ec", + ErasureCodingPolicyManager.getSystemDefaultPolicy().getName()); } @After diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteStripedFileWithFailure.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteStripedFileWithFailure.java index b89e16916a..23a9821490 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteStripedFileWithFailure.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteStripedFileWithFailure.java @@ -59,7 +59,8 @@ public class TestWriteStripedFileWithFailure { public void setup() throws IOException { conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize); cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build(); - cluster.getFileSystem().getClient().setErasureCodingPolicy("/", null); + cluster.getFileSystem().getClient().setErasureCodingPolicy("/", + ErasureCodingPolicyManager.getSystemDefaultPolicy().getName()); fs = cluster.getFileSystem(); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java index 1906d74285..70aa4e08ca 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java @@ -1940,7 +1940,8 @@ private void doTestBalancerWithStripedFile(Configuration conf) throws Exception cluster.waitActive(); client = NameNodeProxies.createProxy(conf, cluster.getFileSystem(0).getUri(), ClientProtocol.class).getProxy(); - client.setErasureCodingPolicy("/", null); + client.setErasureCodingPolicy("/", + ErasureCodingPolicyManager.getSystemDefaultPolicy().getName()); long totalCapacity = sum(capacities); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFSStriped.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFSStriped.java index 834df6f742..20c3accbd3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFSStriped.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFSStriped.java @@ -83,8 +83,8 @@ public void testRead() throws Exception { .nameNodeHttpPort(ServerSocketUtil.getPort(19870, 100)) .numDataNodes(numDNs) .build(); - cluster.getFileSystem().getClient() - .setErasureCodingPolicy("/", null); + cluster.getFileSystem().getClient().setErasureCodingPolicy("/", + ErasureCodingPolicyManager.getSystemDefaultPolicy().getName()); try { cluster.waitActive(); doTestRead(conf, cluster, true); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReconstructStripedBlocksWithRackAwareness.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReconstructStripedBlocksWithRackAwareness.java index ace8732a84..832876e449 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReconstructStripedBlocksWithRackAwareness.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReconstructStripedBlocksWithRackAwareness.java @@ -150,7 +150,8 @@ public void testReconstructForNotEnoughRacks() throws Exception { .numDataNodes(hosts.length).build(); cluster.waitActive(); fs = cluster.getFileSystem(); - fs.setErasureCodingPolicy(new Path("/"), null); + fs.setErasureCodingPolicy(new Path("/"), + ErasureCodingPolicyManager.getSystemDefaultPolicy().getName()); FSNamesystem fsn = cluster.getNamesystem(); BlockManager bm = fsn.getBlockManager(); @@ -220,7 +221,8 @@ public void testChooseExcessReplicasToDelete() throws Exception { .numDataNodes(hosts.length).build(); cluster.waitActive(); fs = cluster.getFileSystem(); - fs.setErasureCodingPolicy(new Path("/"), null); + fs.setErasureCodingPolicy(new Path("/"), + ErasureCodingPolicyManager.getSystemDefaultPolicy().getName()); MiniDFSCluster.DataNodeProperties lastHost = stopDataNode( hosts[hosts.length - 1]); @@ -273,7 +275,8 @@ public void testReconstructionWithDecommission() throws Exception { .numDataNodes(hostNames.length).build(); cluster.waitActive(); fs = cluster.getFileSystem(); - fs.setErasureCodingPolicy(new Path("/"), null); + fs.setErasureCodingPolicy(new Path("/"), + ErasureCodingPolicyManager.getSystemDefaultPolicy().getName()); final BlockManager bm = cluster.getNamesystem().getBlockManager(); final DatanodeManager dm = bm.getDatanodeManager(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestSequentialBlockGroupId.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestSequentialBlockGroupId.java index 10ea01f2da..7920c59307 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestSequentialBlockGroupId.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestSequentialBlockGroupId.java @@ -88,8 +88,8 @@ public void setup() throws Exception { blockGrpIdGenerator = cluster.getNamesystem().getBlockManager() .getBlockIdManager().getBlockGroupIdGenerator(); fs.mkdirs(ecDir); - cluster.getFileSystem().getClient() - .setErasureCodingPolicy("/ecDir", null); + cluster.getFileSystem().getClient().setErasureCodingPolicy("/ecDir", + ErasureCodingPolicyManager.getSystemDefaultPolicy().getName()); } @After diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeErasureCodingMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeErasureCodingMetrics.java index 7e64214dd9..7036c7a45b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeErasureCodingMetrics.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeErasureCodingMetrics.java @@ -75,7 +75,8 @@ public void setup() throws IOException { conf.setInt(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY, 1); cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build(); cluster.waitActive(); - cluster.getFileSystem().getClient().setErasureCodingPolicy("/", null); + cluster.getFileSystem().getClient().setErasureCodingPolicy("/", + ErasureCodingPolicyManager.getSystemDefaultPolicy().getName()); fs = cluster.getFileSystem(); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestMover.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestMover.java index d565548e91..a403ff4bb0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestMover.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestMover.java @@ -537,7 +537,8 @@ public void testMoverWithStripedFile() throws Exception { client.setStoragePolicy(barDir, HdfsConstants.HOT_STORAGE_POLICY_NAME); // set an EC policy on "/bar" directory - client.setErasureCodingPolicy(barDir, null); + client.setErasureCodingPolicy(barDir, + ErasureCodingPolicyManager.getSystemDefaultPolicy().getName()); // write file to barDir final String fooFile = "/bar/foo"; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddOverReplicatedStripedBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddOverReplicatedStripedBlocks.java index c8b2556457..670efd6955 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddOverReplicatedStripedBlocks.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddOverReplicatedStripedBlocks.java @@ -81,7 +81,8 @@ public void setup() throws IOException { cluster.waitActive(); fs = cluster.getFileSystem(); fs.mkdirs(dirPath); - fs.getClient().setErasureCodingPolicy(dirPath.toString(), null); + fs.getClient().setErasureCodingPolicy(dirPath.toString(), + ErasureCodingPolicyManager.getSystemDefaultPolicy().getName()); } @After diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddStripedBlockInFBR.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddStripedBlockInFBR.java index 6e02372a7f..87fbcc63c5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddStripedBlockInFBR.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddStripedBlockInFBR.java @@ -87,7 +87,8 @@ public void testAddBlockInFullBlockReport() throws Exception { final Path repDir = new Path("/rep"); dfs.mkdirs(ecDir); dfs.mkdirs(repDir); - dfs.getClient().setErasureCodingPolicy(ecDir.toString(), null); + dfs.getClient().setErasureCodingPolicy(ecDir.toString(), + ErasureCodingPolicyManager.getSystemDefaultPolicy().getName()); // create several non-EC files and one EC file final Path[] repFiles = new Path[groupSize]; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddStripedBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddStripedBlocks.java index ffd3fa77c8..2df1aa4985 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddStripedBlocks.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddStripedBlocks.java @@ -86,7 +86,8 @@ public void setup() throws IOException { .numDataNodes(groupSize).build(); cluster.waitActive(); dfs = cluster.getFileSystem(); - dfs.getClient().setErasureCodingPolicy("/", null); + dfs.getClient().setErasureCodingPolicy("/", ErasureCodingPolicyManager + .getSystemDefaultPolicy().getName()); } @After diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java index a57a936b08..72d76b77ef 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java @@ -479,7 +479,7 @@ public void testAddNewStripedBlock() throws IOException{ //set the storage policy of the directory fs.mkdir(new Path(testDir), new FsPermission("755")); fs.getClient().getNamenode().setErasureCodingPolicy( - testDir, testECPolicy); + testDir, testECPolicy.getName()); // Create a file with striped block Path p = new Path(testFilePath); @@ -552,7 +552,7 @@ public void testUpdateStripedBlocks() throws IOException{ //set the storage policy of the directory fs.mkdir(new Path(testDir), new FsPermission("755")); fs.getClient().getNamenode().setErasureCodingPolicy( - testDir, testECPolicy); + testDir, testECPolicy.getName()); //create a file with striped blocks Path p = new Path(testFilePath); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java index 179e297580..ae15491786 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java @@ -156,7 +156,7 @@ private void testPersistHelper(Configuration conf) throws IOException { private void testSaveAndLoadStripedINodeFile(FSNamesystem fsn, Configuration conf, boolean isUC) throws IOException{ // Construct an INode with StripedBlock for saving and loading - fsn.setErasureCodingPolicy("/", testECPolicy, false); + fsn.setErasureCodingPolicy("/", testECPolicy.getName(), false); long id = 123456789; byte[] name = "testSaveAndLoadInodeFile_testfile".getBytes(); PermissionStatus permissionStatus = new PermissionStatus("testuser_a", @@ -472,8 +472,8 @@ public void testSupportBlockGroup() throws Exception { // Create directories and files fs.mkdirs(parentDir); fs.mkdirs(childDir); - fs.setErasureCodingPolicy(parentDir, testECPolicy); - fs.setErasureCodingPolicy(childDir, ec32Policy); + fs.setErasureCodingPolicy(parentDir, testECPolicy.getName()); + fs.setErasureCodingPolicy(childDir, ec32Policy.getName()); Path file_10_4 = new Path(parentDir, "striped_file_10_4"); Path file_3_2 = new Path(childDir, "striped_file_3_2"); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java index c02e880055..76c5378062 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java @@ -690,7 +690,7 @@ public void testFsckOpenECFiles() throws Exception { util.createFiles(fs, topDir); // set topDir to EC when it has replicated files cluster.getFileSystem().getClient().setErasureCodingPolicy( - topDir, ecPolicy); + topDir, ecPolicy.getName()); // create a new file under topDir DFSTestUtil.createFile(fs, new Path(topDir, "ecFile"), 1024, (short) 1, 0L); @@ -2307,7 +2307,8 @@ public void testFsckCorruptECFile() throws Exception { // create file Path ecDirPath = new Path("/striped"); fs.mkdir(ecDirPath, FsPermission.getDirDefault()); - fs.getClient().setErasureCodingPolicy(ecDirPath.toString(), null); + fs.getClient().setErasureCodingPolicy(ecDirPath.toString(), + ErasureCodingPolicyManager.getSystemDefaultPolicy().getName()); Path file = new Path(ecDirPath, "corrupted"); final int length = cellSize * dataBlocks; final byte[] bytes = StripedFileTestUtil.generateBytes(length); @@ -2372,7 +2373,8 @@ public void testFsckMissingECFile() throws Exception { // create file Path ecDirPath = new Path("/striped"); fs.mkdir(ecDirPath, FsPermission.getDirDefault()); - fs.getClient().setErasureCodingPolicy(ecDirPath.toString(), null); + fs.getClient().setErasureCodingPolicy(ecDirPath.toString(), + ErasureCodingPolicyManager.getSystemDefaultPolicy().getName()); Path file = new Path(ecDirPath, "missing"); final int length = cellSize * dataBlocks; final byte[] bytes = StripedFileTestUtil.generateBytes(length); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java index ef1e6e4368..037fd4025b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java @@ -738,7 +738,8 @@ public void testVerifyMissingBlockGroupsMetrics() throws Exception { // create file Path ecDirPath = new Path("/striped"); fs.mkdir(ecDirPath, FsPermission.getDirDefault()); - fs.getClient().setErasureCodingPolicy(ecDirPath.toString(), null); + fs.getClient().setErasureCodingPolicy(ecDirPath.toString(), + ErasureCodingPolicyManager.getSystemDefaultPolicy().getName()); Path file = new Path(ecDirPath, "corrupted"); final int length = cellSize * dataBlocks; final byte[] bytes = StripedFileTestUtil.generateBytes(length); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestQuotaWithStripedBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestQuotaWithStripedBlocks.java index 9ec34d25b6..326ddc84c6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestQuotaWithStripedBlocks.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestQuotaWithStripedBlocks.java @@ -71,7 +71,8 @@ public void setUp() throws IOException { dfs = cluster.getFileSystem(); dfs.mkdirs(ecDir); - dfs.getClient().setErasureCodingPolicy(ecDir.toString(), ecPolicy); + dfs.getClient() + .setErasureCodingPolicy(ecDir.toString(), ecPolicy.getName()); dfs.setQuota(ecDir, Long.MAX_VALUE - 1, DISK_QUOTA); dfs.setQuotaByStorageType(ecDir, StorageType.DISK, DISK_QUOTA); dfs.setStoragePolicy(ecDir, HdfsConstants.HOT_STORAGE_POLICY_NAME); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestReconstructStripedBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestReconstructStripedBlocks.java index cf1585c07d..5e4a9db39d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestReconstructStripedBlocks.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestReconstructStripedBlocks.java @@ -201,7 +201,8 @@ public void test2RecoveryTasksForSameBlockGroup() throws Exception { cluster.waitActive(); DistributedFileSystem fs = cluster.getFileSystem(); BlockManager bm = cluster.getNamesystem().getBlockManager(); - fs.getClient().setErasureCodingPolicy("/", null); + fs.getClient().setErasureCodingPolicy("/", + ErasureCodingPolicyManager.getSystemDefaultPolicy().getName()); int fileLen = dataBlocks * blockSize; Path p = new Path("/test2RecoveryTasksForSameBlockGroup"); final byte[] data = new byte[fileLen]; @@ -266,7 +267,8 @@ public void testCountLiveReplicas() throws Exception { try { fs.mkdirs(dirPath); - fs.setErasureCodingPolicy(dirPath, null); + fs.setErasureCodingPolicy(dirPath, + ErasureCodingPolicyManager.getSystemDefaultPolicy().getName()); DFSTestUtil.createFile(fs, filePath, cellSize * dataBlocks * 2, (short) 1, 0L); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStripedINodeFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStripedINodeFile.java index 03bd124e82..a712498cef 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStripedINodeFile.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStripedINodeFile.java @@ -307,7 +307,8 @@ public void testDeleteOp() throws Exception { dfs.mkdirs(ecDir); // set erasure coding policy - dfs.setErasureCodingPolicy(ecDir, null); + dfs.setErasureCodingPolicy(ecDir, + ErasureCodingPolicyManager.getSystemDefaultPolicy().getName()); DFSTestUtil.createFile(dfs, ecFile, len, (short) 1, 0xFEED); DFSTestUtil.createFile(dfs, contiguousFile, len, (short) 1, 0xFEED); final FSDirectory fsd = fsn.getFSDirectory(); @@ -408,7 +409,8 @@ public void testUnsuitableStoragePoliciesWithECStripedMode() client.mkdirs(fooDir, new FsPermission((short) 777), true); client.setStoragePolicy(fooDir, HdfsConstants.ONESSD_STORAGE_POLICY_NAME); // set an EC policy on "/foo" directory - client.setErasureCodingPolicy(fooDir, null); + client.setErasureCodingPolicy(fooDir, + ErasureCodingPolicyManager.getSystemDefaultPolicy().getName()); // write file to fooDir final String barFile = "/foo/bar"; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java index 053b8567d5..b587017b91 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java @@ -236,7 +236,8 @@ public static void createOriginalFSImage() throws IOException { ErasureCodingPolicy ecPolicy = ErasureCodingPolicyManager.getPolicyByPolicyID( HdfsConstants.XOR_2_1_POLICY_ID); - hdfs.getClient().setErasureCodingPolicy(ecDir.toString(), ecPolicy); + hdfs.getClient().setErasureCodingPolicy(ecDir.toString(), + ecPolicy.getName()); writtenFiles.put(ecDir.toString(), hdfs.getFileStatus(ecDir)); // Create an empty Erasure Coded file diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewerWithStripedBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewerWithStripedBlocks.java index db21217fc0..e7794d6c9e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewerWithStripedBlocks.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewerWithStripedBlocks.java @@ -63,7 +63,8 @@ public void setup() throws IOException { conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize); cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build(); cluster.waitActive(); - cluster.getFileSystem().getClient().setErasureCodingPolicy("/", null); + cluster.getFileSystem().getClient().setErasureCodingPolicy("/", + ErasureCodingPolicyManager.getSystemDefaultPolicy().getName()); fs = cluster.getFileSystem(); Path eczone = new Path("/eczone"); fs.mkdirs(eczone); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml index 9b9003a541..278963a72d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml @@ -359,7 +359,7 @@ SubstringComparator - Policy 'invalidpolicy' does not match any of the supported policies. Please select any one of [ + Policy 'invalidpolicy' does not match any supported erasure coding policies. diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/TestDFSIO.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/TestDFSIO.java index d218169280..2eb4b29be7 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/TestDFSIO.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/TestDFSIO.java @@ -989,7 +989,7 @@ void createAndEnableECOnPath(FileSystem fs, Path path) ((DistributedFileSystem) fs).getAllErasureCodingPolicies(); for (ErasureCodingPolicy ec : list) { if (erasureCodePolicyName.equals(ec.getName())) { - ((DistributedFileSystem) fs).setErasureCodingPolicy(path, ec); + ((DistributedFileSystem) fs).setErasureCodingPolicy(path, ec.getName()); LOG.info("enable erasureCodePolicy = " + erasureCodePolicyName + " on " + path.toString()); break;