big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-11428. Change setErasureCodingPolicy to take a required string EC policy name. Contributed by Andrew Wang.

Browse Source
This commit is contained in:
Rakesh Radhakrishnan 2017-03-01 16:06:02 +05:30
parent dcd03df9f9
commit 82ef9accaf
49 changed files with 152 additions and 132 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
View File

@ -2611,12 +2611,12 @@ public RemoteIterator<EncryptionZone> listEncryptionZones()
}
public void setErasureCodingPolicy(String src, ErasureCodingPolicy ecPolicy)
public void setErasureCodingPolicy(String src, String ecPolicyName)
throws IOException {
checkOpen();
try (TraceScope ignored =
newPathTraceScope("setErasureCodingPolicy", src)) {
namenode.setErasureCodingPolicy(src, ecPolicy);
namenode.setErasureCodingPolicy(src, ecPolicyName);
} catch (RemoteException re) {
throw re.unwrapRemoteException(AccessControlException.class,
SafeModeException.class,

11
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
View File

@ -2397,17 +2397,18 @@ public DFSInotifyEventInputStream getInotifyEventStream(long lastReadTxid)
* Set the source path to the specified erasure coding policy.
*
* @param path The directory to set the policy
* @param ecPolicy The erasure coding policy. If not specified default will
* be used.
* @param ecPolicyName The erasure coding policy name.
* @throws IOException
*/
public void setErasureCodingPolicy(final Path path,
final ErasureCodingPolicy ecPolicy) throws IOException {
final String ecPolicyName) throws IOException {
Path absF = fixRelativePart(path);
Preconditions.checkNotNull(ecPolicyName, "Erasure coding policy cannot be" +
" null.");
new FileSystemLinkResolver<Void>() {
@Override
public Void doCall(final Path p) throws IOException {
dfs.setErasureCodingPolicy(getPathName(p), ecPolicy);
dfs.setErasureCodingPolicy(getPathName(p), ecPolicyName);
return null;
}
@ -2415,7 +2416,7 @@ public Void doCall(final Path p) throws IOException {
public Void next(final FileSystem fs, final Path p) throws IOException {
if (fs instanceof DistributedFileSystem) {
DistributedFileSystem myDfs = (DistributedFileSystem) fs;
myDfs.setErasureCodingPolicy(p, ecPolicy);
myDfs.setErasureCodingPolicy(p, ecPolicyName);
return null;
}
throw new UnsupportedOperationException(

10
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java
View File

@ -465,13 +465,15 @@ public Collection<? extends BlockStoragePolicySpi> getAllStoragePolicies()
* Set the source path to the specified erasure coding policy.
*
* @param path The source path referring to a directory.
* @param ecPolicy The erasure coding policy for the directory.
* If null, the default will be used.
* @param ecPolicyName The erasure coding policy name for the directory.
*
* @throws IOException
* @throws HadoopIllegalArgumentException if the specified EC policy is not
* enabled on the cluster
*/
public void setErasureCodingPolicy(final Path path,
final ErasureCodingPolicy ecPolicy) throws IOException {
dfs.setErasureCodingPolicy(path, ecPolicy);
final String ecPolicyName) throws IOException {
dfs.setErasureCodingPolicy(path, ecPolicyName);
}
/**

5
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
View File

@ -1510,11 +1510,10 @@ List<XAttr> listXAttrs(String src)
/**
* Set an erasure coding policy on a specified path.
* @param src The path to set policy on.
* @param ecPolicy The erasure coding policy. If null, default policy will
* be used
* @param ecPolicyName The erasure coding policy name.
*/
@AtMostOnce
void setErasureCodingPolicy(String src, ErasureCodingPolicy ecPolicy)
void setErasureCodingPolicy(String src, String ecPolicyName)
throws IOException;
/**

6
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java
View File

@ -1459,14 +1459,12 @@ public BatchedEntries<EncryptionZone> listEncryptionZones(long id)
}
@Override
public void setErasureCodingPolicy(String src, ErasureCodingPolicy ecPolicy)
public void setErasureCodingPolicy(String src, String ecPolicyName)
throws IOException {
final SetErasureCodingPolicyRequestProto.Builder builder =
SetErasureCodingPolicyRequestProto.newBuilder();
builder.setSrc(src);
if (ecPolicy != null) {
builder.setEcPolicy(PBHelperClient.convertErasureCodingPolicy(ecPolicy));
}
builder.setEcPolicyName(ecPolicyName);
SetErasureCodingPolicyRequestProto req = builder.build();
try {
rpcProxy.setErasureCodingPolicy(null, req);

2
hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/erasurecoding.proto
View File

@ -25,7 +25,7 @@ import "hdfs.proto";
message SetErasureCodingPolicyRequestProto {
required string src = 1;
optional ErasureCodingPolicyProto ecPolicy = 2;
required string ecPolicyName = 2;
}
message SetErasureCodingPolicyResponseProto {

4
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
View File

@ -1442,9 +1442,7 @@ public SetErasureCodingPolicyResponseProto setErasureCodingPolicy(
RpcController controller, SetErasureCodingPolicyRequestProto req)
throws ServiceException {
try {
ErasureCodingPolicy ecPolicy = req.hasEcPolicy() ?
PBHelperClient.convertErasureCodingPolicy(req.getEcPolicy()) : null;
server.setErasureCodingPolicy(req.getSrc(), ecPolicy);
server.setErasureCodingPolicy(req.getSrc(), req.getEcPolicyName());
return SetErasureCodingPolicyResponseProto.newBuilder().build();
} catch (IOException e) {
throw new ServiceException(e);

13
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java
View File

@ -59,14 +59,16 @@ private FSDirErasureCodingOp() {}
*
* @param fsn The namespace
* @param srcArg The path of the target directory.
* @param ecPolicy The erasure coding policy to set on the target directory.
* @param ecPolicyName The erasure coding policy name to set on the target
* directory.
* @param logRetryCache whether to record RPC ids in editlog for retry
* cache rebuilding
* @return {@link HdfsFileStatus}
* @throws IOException
* @throws HadoopIllegalArgumentException if the policy is not enabled
*/
static HdfsFileStatus setErasureCodingPolicy(final FSNamesystem fsn,
final String srcArg, final ErasureCodingPolicy ecPolicy,
final String srcArg, final String ecPolicyName,
final boolean logRetryCache) throws IOException {
assert fsn.hasWriteLock();
@ -78,6 +80,13 @@ static HdfsFileStatus setErasureCodingPolicy(final FSNamesystem fsn,
List<XAttr> xAttrs;
fsd.writeLock();
try {
ErasureCodingPolicy ecPolicy = fsn.getErasureCodingPolicyManager()
.getPolicyByName(ecPolicyName);
if (ecPolicy == null) {
throw new HadoopIllegalArgumentException("Policy '" +
ecPolicyName + "' does not match any supported erasure coding " +
"policies.");
}
iip = fsd.resolvePath(pc, src, DirOp.WRITE_LINK);
src = iip.getPath();
xAttrs = setErasureCodingPolicyXAttr(fsn, iip, ecPolicy);

9
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
View File

@ -6772,13 +6772,14 @@ BatchedListEntries<EncryptionZone> listEncryptionZones(long prevId)
/**
* Set an erasure coding policy on the given path.
* @param srcArg The path of the target directory.
* @param ecPolicy The erasure coding policy to set on the target directory.
* @param ecPolicyName The erasure coding policy to set on the target
* directory.
* @throws AccessControlException if the caller is not the superuser.
* @throws UnresolvedLinkException if the path can't be resolved.
* @throws SafeModeException if the Namenode is in safe mode.
*/
void setErasureCodingPolicy(final String srcArg, final ErasureCodingPolicy
ecPolicy, final boolean logRetryCache) throws IOException,
void setErasureCodingPolicy(final String srcArg, final String ecPolicyName,
final boolean logRetryCache) throws IOException,
UnresolvedLinkException, SafeModeException, AccessControlException {
final String operationName = "setErasureCodingPolicy";
checkSuperuserPrivilege();
@ -6790,7 +6791,7 @@ void setErasureCodingPolicy(final String srcArg, final ErasureCodingPolicy
checkOperation(OperationCategory.WRITE);
checkNameNodeSafeMode("Cannot set erasure coding policy on " + srcArg);
resultingStat = FSDirErasureCodingOp.setErasureCodingPolicy(this,
srcArg, ecPolicy, logRetryCache);
srcArg, ecPolicyName, logRetryCache);
success = true;
} catch (AccessControlException ace) {
logAuditEvent(success, operationName, srcArg, null,

4
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
View File

@ -2018,7 +2018,7 @@ public BatchedEntries<EncryptionZone> listEncryptionZones(
}
@Override // ClientProtocol
public void setErasureCodingPolicy(String src, ErasureCodingPolicy ecPolicy)
public void setErasureCodingPolicy(String src, String ecPolicyName)
throws IOException {
checkNNStartup();
final CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache);
@ -2027,7 +2027,7 @@ public void setErasureCodingPolicy(String src, ErasureCodingPolicy ecPolicy)
}
boolean success = false;
try {
namesystem.setErasureCodingPolicy(src, ecPolicy, cacheEntry != null);
namesystem.setErasureCodingPolicy(src, ecPolicyName, cacheEntry != null);
success = true;
} finally {
RetryCache.setState(cacheEntry, success);

26
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java
View File

@ -28,7 +28,6 @@
import org.apache.hadoop.util.ToolRunner;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.LinkedList;
@ -227,30 +226,7 @@ public int run(Configuration conf, List<String> args) throws IOException {
final Path p = new Path(path);
final DistributedFileSystem dfs = AdminHelper.getDFS(p.toUri(), conf);
try {
ErasureCodingPolicy ecPolicy = null;
ErasureCodingPolicy[] ecPolicies =
dfs.getClient().getErasureCodingPolicies();
for (ErasureCodingPolicy policy : ecPolicies) {
if (ecPolicyName.equals(policy.getName())) {
ecPolicy = policy;
break;
}
}
if (ecPolicy == null) {
StringBuilder sb = new StringBuilder();
sb.append("Policy '");
sb.append(ecPolicyName);
sb.append("' does not match any of the supported policies.");
sb.append(" Please select any one of ");
List<String> ecPolicyNames = new ArrayList<String>();
for (ErasureCodingPolicy policy : ecPolicies) {
ecPolicyNames.add(policy.getName());
}
sb.append(ecPolicyNames);
System.err.println(sb.toString());
return 3;
}
dfs.setErasureCodingPolicy(p, ecPolicy);
dfs.setErasureCodingPolicy(p, ecPolicyName);
System.out.println("Set erasure coding policy " + ecPolicyName +
" on " + path);
} catch (Exception e) {

6
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
View File

@ -136,6 +136,7 @@
import org.apache.hadoop.hdfs.server.datanode.SimulatedFSDataset;
import org.apache.hadoop.hdfs.server.datanode.TestTransferRbw;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi;
import org.apache.hadoop.hdfs.server.namenode.ErasureCodingPolicyManager;
import org.apache.hadoop.hdfs.server.namenode.FSDirectory;
import org.apache.hadoop.hdfs.server.namenode.FSEditLog;
import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
@ -1898,7 +1899,7 @@ public static void createStripedFile(MiniDFSCluster cluster, Path file,
Path dir, int numBlocks, int numStripesPerBlk, boolean toMkdir)
throws Exception {
createStripedFile(cluster, file, dir, numBlocks, numStripesPerBlk,
toMkdir, null);
toMkdir, ErasureCodingPolicyManager.getSystemDefaultPolicy());
}
/**
@ -1922,7 +1923,8 @@ public static void createStripedFile(MiniDFSCluster cluster, Path file,
assert dir != null;
dfs.mkdirs(dir);
try {
dfs.getClient().setErasureCodingPolicy(dir.toString(), ecPolicy);
dfs.getClient()
.setErasureCodingPolicy(dir.toString(), ecPolicy.getName());
} catch (IOException e) {
if (!e.getMessage().contains("non-empty directory")) {
throw e;

3
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/ErasureCodeBenchmarkThroughput.java
View File

@ -192,7 +192,8 @@ private void setUpDir() throws IOException {
}
if (!dfs.exists(ecPath)) {
dfs.mkdirs(ecPath);
dfs.getClient().setErasureCodingPolicy(ecPath.toString(), ecPolicy);
dfs.getClient()
.setErasureCodingPolicy(ecPath.toString(), ecPolicy.getName());
} else {
Preconditions.checkArgument(
dfs.getClient().

3
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java
View File

@ -108,7 +108,8 @@ public void setup() throws IOException {
}
fs = cluster.getFileSystem();
fs.mkdirs(dirPath);
fs.getClient().setErasureCodingPolicy(dirPath.toString(), ecPolicy);
fs.getClient()
.setErasureCodingPolicy(dirPath.toString(), ecPolicy.getName());
}
@After

3
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java
View File

@ -89,7 +89,8 @@ public void setup() throws IOException {
NativeRSRawErasureCoderFactory.class.getCanonicalName());
}
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build();
cluster.getFileSystem().getClient().setErasureCodingPolicy("/", ecPolicy);
cluster.getFileSystem().getClient().setErasureCodingPolicy("/", ecPolicy
.getName());
fs = cluster.getFileSystem();
}

2
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java
View File

@ -221,7 +221,7 @@ private void setup(Configuration conf) throws IOException {
cluster.waitActive();
dfs = cluster.getFileSystem();
dfs.mkdirs(dir);
dfs.setErasureCodingPolicy(dir, ecPolicy);
dfs.setErasureCodingPolicy(dir, ecPolicy.getName());
}
private void tearDown() {

3
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommissionWithStriped.java
View File

@ -142,7 +142,8 @@ public void setup() throws IOException {
client = getDfsClient(cluster.getNameNode(0), conf);
dfs.mkdirs(ecDir);
dfs.setErasureCodingPolicy(ecDir, null);
dfs.setErasureCodingPolicy(ecDir,
ErasureCodingPolicyManager.getSystemDefaultPolicy().getName());
}
@After

37
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java
View File

@ -47,6 +47,8 @@ public class TestErasureCodingPolicies {
private MiniDFSCluster cluster;
private DistributedFileSystem fs;
private static final int BLOCK_SIZE = 1024;
private static final ErasureCodingPolicy EC_POLICY =
ErasureCodingPolicyManager.getSystemDefaultPolicy();
private FSNamesystem namesystem;
@Before
@ -80,7 +82,8 @@ public void testReplicatedFileUnderECDir() throws IOException {
DFSTestUtil.createFile(fs, replicatedFile, 0, (short) 3, 0L);
// set ec policy on dir
fs.setErasureCodingPolicy(dir, null);
fs.setErasureCodingPolicy(dir,
ErasureCodingPolicyManager.getSystemDefaultPolicy().getName());
// create a file which should be using ec
final Path ecSubDir = new Path(dir, "ecSubDir");
final Path ecFile = new Path(ecSubDir, "ecFile");
@ -132,7 +135,7 @@ public void testBasicSetECPolicy()
fs.mkdir(testDir, FsPermission.getDirDefault());
/* Normal creation of an erasure coding directory */
fs.getClient().setErasureCodingPolicy(testDir.toString(), null);
fs.setErasureCodingPolicy(testDir, EC_POLICY.getName());
/* Verify files under the directory are striped */
final Path ECFilePath = new Path(testDir, "foo");
@ -148,7 +151,7 @@ public void testBasicSetECPolicy()
fs.mkdir(notEmpty, FsPermission.getDirDefault());
final Path oldFile = new Path(notEmpty, "old");
fs.create(oldFile);
fs.getClient().setErasureCodingPolicy(notEmpty.toString(), null);
fs.setErasureCodingPolicy(notEmpty, EC_POLICY.getName());
final Path newFile = new Path(notEmpty, "new");
fs.create(newFile);
INode oldInode = namesystem.getFSDirectory().getINode(oldFile.toString());
@ -160,10 +163,10 @@ public void testBasicSetECPolicy()
final Path dir1 = new Path("/dir1");
final Path dir2 = new Path(dir1, "dir2");
fs.mkdir(dir1, FsPermission.getDirDefault());
fs.getClient().setErasureCodingPolicy(dir1.toString(), null);
fs.setErasureCodingPolicy(dir1, EC_POLICY.getName());
fs.mkdir(dir2, FsPermission.getDirDefault());
try {
fs.getClient().setErasureCodingPolicy(dir2.toString(), null);
fs.setErasureCodingPolicy(dir2, EC_POLICY.getName());
} catch (IOException e) {
fail("Nested erasure coding policies are supported");
}
@ -172,7 +175,7 @@ public void testBasicSetECPolicy()
final Path fPath = new Path("/file");
fs.create(fPath);
try {
fs.getClient().setErasureCodingPolicy(fPath.toString(), null);
fs.setErasureCodingPolicy(fPath, EC_POLICY.getName());
fail("Erasure coding policy on file");
} catch (IOException e) {
assertExceptionContains("erasure coding policy for a file", e);
@ -185,8 +188,8 @@ public void testMoveValidity() throws IOException, InterruptedException {
final Path dstECDir = new Path("/dstEC");
fs.mkdir(srcECDir, FsPermission.getDirDefault());
fs.mkdir(dstECDir, FsPermission.getDirDefault());
fs.getClient().setErasureCodingPolicy(srcECDir.toString(), null);
fs.getClient().setErasureCodingPolicy(dstECDir.toString(), null);
fs.setErasureCodingPolicy(srcECDir, EC_POLICY.getName());
fs.setErasureCodingPolicy(dstECDir, EC_POLICY.getName());
final Path srcFile = new Path(srcECDir, "foo");
fs.create(srcFile);
@ -220,7 +223,8 @@ public void testMoveValidity() throws IOException, InterruptedException {
public void testReplication() throws IOException {
final Path testDir = new Path("/ec");
fs.mkdir(testDir, FsPermission.getDirDefault());
fs.setErasureCodingPolicy(testDir, null);
fs.setErasureCodingPolicy(testDir,
ErasureCodingPolicyManager.getSystemDefaultPolicy().getName());
final Path fooFile = new Path(testDir, "foo");
// create ec file with replication=0
fs.create(fooFile, FsPermission.getFileDefault(), true,
@ -241,8 +245,9 @@ public void testGetErasureCodingPolicyWithSystemDefaultECPolicy() throws Excepti
// dir EC policy should be null
assertNull(fs.getClient().getFileInfo(src).getErasureCodingPolicy());
// dir EC policy after setting
fs.getClient().setErasureCodingPolicy(src, null); //Default one will be used.
ErasureCodingPolicy sysDefaultECPolicy = ErasureCodingPolicyManager.getSystemDefaultPolicy();
ErasureCodingPolicy sysDefaultECPolicy =
ErasureCodingPolicyManager.getSystemDefaultPolicy();
fs.getClient().setErasureCodingPolicy(src, sysDefaultECPolicy.getName());
verifyErasureCodingInfo(src, sysDefaultECPolicy);
fs.create(new Path(ecDir, "child1")).close();
// verify for the files in ec dir
@ -263,7 +268,7 @@ public void testGetErasureCodingPolicy() throws Exception {
// dir ECInfo before being set
assertNull(fs.getClient().getFileInfo(src).getErasureCodingPolicy());
// dir ECInfo after set
fs.getClient().setErasureCodingPolicy(src, usingECPolicy);
fs.getClient().setErasureCodingPolicy(src, usingECPolicy.getName());
verifyErasureCodingInfo(src, usingECPolicy);
fs.create(new Path(ecDir, "child1")).close();
// verify for the files in ec dir
@ -291,12 +296,12 @@ public void testCreationErasureCodingZoneWithInvalidPolicy()
final Path ecDir = new Path(src);
try {
fs.mkdir(ecDir, FsPermission.getDirDefault());
fs.getClient().setErasureCodingPolicy(src, ecPolicy);
fs.getClient().setErasureCodingPolicy(src, ecPolicy.getName());
fail("HadoopIllegalArgumentException should be thrown for"
+ "setting an invalid erasure coding policy");
} catch (Exception e) {
assertExceptionContains("Policy [ RS-4-2-128k ] does not match " +
"any of the supported policies",e);
assertExceptionContains("Policy 'RS-4-2-128k' does not match " +
"any supported erasure coding policies",e);
}
}
@ -338,7 +343,7 @@ public void testMultiplePoliciesCoExist() throws Exception {
for (ErasureCodingPolicy policy : sysPolicies) {
Path dir = new Path("/policy_" + policy.getId());
fs.mkdir(dir, FsPermission.getDefault());
fs.setErasureCodingPolicy(dir, policy);
fs.setErasureCodingPolicy(dir, policy.getName());
Path file = new Path(dir, "child");
fs.create(file).close();
assertEquals(policy, fs.getErasureCodingPolicy(file));

10
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicyWithSnapshot.java
View File

@ -75,7 +75,7 @@ public void testSnapshotsOnErasureCodingDirsParentDir() throws Exception {
fs.mkdirs(ecDir);
fs.allowSnapshot(ecDirParent);
// set erasure coding policy
fs.setErasureCodingPolicy(ecDir, sysDefaultPolicy);
fs.setErasureCodingPolicy(ecDir, sysDefaultPolicy.getName());
DFSTestUtil.createFile(fs, ecFile, len, (short) 1, 0xFEED);
String contents = DFSTestUtil.readFile(fs, ecFile);
final Path snap1 = fs.createSnapshot(ecDirParent, "snap1");
@ -93,7 +93,7 @@ public void testSnapshotsOnErasureCodingDirsParentDir() throws Exception {
fs.getErasureCodingPolicy(snap2ECDir));
// Make dir again with system default ec policy
fs.setErasureCodingPolicy(ecDir, sysDefaultPolicy);
fs.setErasureCodingPolicy(ecDir, sysDefaultPolicy.getName());
final Path snap3 = fs.createSnapshot(ecDirParent, "snap3");
final Path snap3ECDir = new Path(snap3, ecDir.getName());
// Check that snap3's ECPolicy has the correct settings
@ -134,7 +134,7 @@ public void testSnapshotsOnErasureCodingDir() throws Exception {
fs.mkdirs(ecDir);
fs.allowSnapshot(ecDir);
fs.setErasureCodingPolicy(ecDir, sysDefaultPolicy);
fs.setErasureCodingPolicy(ecDir, sysDefaultPolicy.getName());
final Path snap1 = fs.createSnapshot(ecDir, "snap1");
assertEquals("Got unexpected erasure coding policy", sysDefaultPolicy,
fs.getErasureCodingPolicy(snap1));
@ -150,7 +150,7 @@ public void testSnapshotsOnErasureCodingDirAfterNNRestart() throws Exception {
fs.allowSnapshot(ecDir);
// set erasure coding policy
fs.setErasureCodingPolicy(ecDir, sysDefaultPolicy);
fs.setErasureCodingPolicy(ecDir, sysDefaultPolicy.getName());
final Path snap1 = fs.createSnapshot(ecDir, "snap1");
ErasureCodingPolicy ecSnap = fs.getErasureCodingPolicy(snap1);
assertEquals("Got unexpected erasure coding policy", sysDefaultPolicy,
@ -182,7 +182,7 @@ public void testCopySnapshotWillNotPreserveErasureCodingPolicy()
fs.allowSnapshot(ecDir);
// set erasure coding policy
fs.setErasureCodingPolicy(ecDir, sysDefaultPolicy);
fs.setErasureCodingPolicy(ecDir, sysDefaultPolicy.getName());
DFSTestUtil.createFile(fs, ecFile, len, (short) 1, 0xFEED);
final Path snap1 = fs.createSnapshot(ecDir, "snap1");

3
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileChecksum.java
View File

@ -81,7 +81,8 @@ public void setup() throws IOException {
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build();
Path ecPath = new Path(ecDir);
cluster.getFileSystem().mkdir(ecPath, FsPermission.getDirDefault());
cluster.getFileSystem().getClient().setErasureCodingPolicy(ecDir, null);
cluster.getFileSystem().getClient().setErasureCodingPolicy(ecDir,
ErasureCodingPolicyManager.getSystemDefaultPolicy().getName());
fs = cluster.getFileSystem();
client = fs.getClient();

2
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatusWithECPolicy.java
View File

@ -73,7 +73,7 @@ public void testFileStatusWithECPolicy() throws Exception {
final ErasureCodingPolicy ecPolicy1 = ErasureCodingPolicyManager.getSystemDefaultPolicy();
// set EC policy on dir
fs.setErasureCodingPolicy(dir, ecPolicy1);
fs.setErasureCodingPolicy(dir, ecPolicy1.getName());
final ErasureCodingPolicy ecPolicy2 = client.getFileInfo(dir.toUri().getPath()).getErasureCodingPolicy();
assertNotNull(ecPolicy2);
assertTrue(ecPolicy1.equals(ecPolicy2));

2
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecoveryStriped.java
View File

@ -94,7 +94,7 @@ public void setup() throws IOException {
cluster.waitActive();
dfs = cluster.getFileSystem();
dfs.mkdirs(dir);
dfs.setErasureCodingPolicy(dir, ecPolicy);
dfs.setErasureCodingPolicy(dir, ecPolicy.getName());
}
@After

3
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReadStripedFileWithDecoding.java
View File

@ -102,7 +102,8 @@ public void setup() throws IOException {
conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_KEY,
false);
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build();
cluster.getFileSystem().getClient().setErasureCodingPolicy("/", null);
cluster.getFileSystem().getClient().setErasureCodingPolicy("/",
ErasureCodingPolicyManager.getSystemDefaultPolicy().getName());
fs = cluster.getFileSystem();
}

2
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReadStripedFileWithMissingBlocks.java
View File

@ -61,7 +61,7 @@ public void setup() throws IOException {
conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_MAX_STREAMS_KEY, 0);
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build();
cluster.getFileSystem().getClient().setErasureCodingPolicy(
"/", ecPolicy);
"/", ecPolicy.getName());
fs = cluster.getFileSystem();
}

3
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReconstructStripedFile.java
View File

@ -107,7 +107,8 @@ public void setup() throws IOException {
cluster.waitActive();
fs = cluster.getFileSystem();
fs.getClient().setErasureCodingPolicy("/", null);
fs.getClient().setErasureCodingPolicy("/",
ErasureCodingPolicyManager.getSystemDefaultPolicy().getName());
List<DataNode> datanodes = cluster.getDataNodes();
for (int i = 0; i < dnNum; i++) {

3
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeModeWithStripedFile.java
View File

@ -63,7 +63,8 @@ public void setup() throws IOException {
conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize);
conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, 100);
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build();
cluster.getFileSystem().getClient().setErasureCodingPolicy("/", null);
cluster.getFileSystem().getClient().setErasureCodingPolicy("/",
ErasureCodingPolicyManager.getSystemDefaultPolicy().getName());
cluster.waitActive();
}

18
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestUnsetAndChangeDirectoryEcPolicy.java
View File

@ -101,7 +101,7 @@ public void testUnsetEcPolicy() throws Exception {
// Test unset a directory which has no EC policy
fs.unsetErasureCodingPolicy(dirPath);
// Set EC policy on directory
fs.setErasureCodingPolicy(dirPath, ecPolicy);
fs.setErasureCodingPolicy(dirPath, ecPolicy.getName());
DFSTestUtil.createFile(fs, ecFilePath, fileLen, (short) 1, 0L);
fs.unsetErasureCodingPolicy(dirPath);
@ -142,12 +142,12 @@ public void testNestedEcPolicy() throws Exception {
.getPolicyByPolicyID(HdfsConstants.RS_3_2_POLICY_ID);
fs.mkdirs(parentDir);
fs.setErasureCodingPolicy(parentDir, ecPolicy);
fs.setErasureCodingPolicy(parentDir, ecPolicy.getName());
fs.mkdirs(childDir);
// Create RS(6,3) EC policy file
DFSTestUtil.createFile(fs, ec63FilePath, fileLen, (short) 1, 0L);
// Set RS(3,2) EC policy on child directory
fs.setErasureCodingPolicy(childDir, ec32Policy);
fs.setErasureCodingPolicy(childDir, ec32Policy.getName());
// Create RS(3,2) EC policy file
DFSTestUtil.createFile(fs, ec32FilePath, fileLen, (short) 1, 0L);
@ -202,7 +202,7 @@ public void testUnsetRootDirEcPolicy() throws Exception {
// Test unset root path which has no EC policy
fs.unsetErasureCodingPolicy(rootPath);
// Set EC policy on root path
fs.setErasureCodingPolicy(rootPath, ecPolicy);
fs.setErasureCodingPolicy(rootPath, ecPolicy.getName());
DFSTestUtil.createFile(fs, ecFilePath, fileLen, (short) 1, 0L);
fs.unsetErasureCodingPolicy(rootPath);
DFSTestUtil.createFile(fs, replicateFilePath, fileLen, (short) 1, 0L);
@ -240,11 +240,11 @@ public void testChangeRootDirEcPolicy() throws Exception {
.getPolicyByPolicyID(HdfsConstants.RS_3_2_POLICY_ID);
fs.unsetErasureCodingPolicy(rootPath);
fs.setErasureCodingPolicy(rootPath, ecPolicy);
fs.setErasureCodingPolicy(rootPath, ecPolicy.getName());
// Create RS(6,3) EC policy file
DFSTestUtil.createFile(fs, ec63FilePath, fileLen, (short) 1, 0L);
// Change EC policy from RS(6,3) to RS(3,2)
fs.setErasureCodingPolicy(rootPath, ec32Policy);
fs.setErasureCodingPolicy(rootPath, ec32Policy.getName());
DFSTestUtil.createFile(fs, ec32FilePath, fileLen, (short) 1, 0L);
// start to check
@ -281,7 +281,7 @@ public void testDifferentReplicaFactor() throws Exception {
final Path replicateFilePath2 = new Path(ecDirPath, "rep_file2");
fs.mkdirs(ecDirPath);
fs.setErasureCodingPolicy(ecDirPath, ecPolicy);
fs.setErasureCodingPolicy(ecDirPath, ecPolicy.getName());
DFSTestUtil.createFile(fs, ecFilePath, fileLen, (short) 1, 0L);
fs.unsetErasureCodingPolicy(ecDirPath);
DFSTestUtil.createFile(fs, replicateFilePath, fileLen, (short) 3, 0L);
@ -328,7 +328,7 @@ public void testNonExistentDir() throws Exception {
// Set EC policy on non-existent directory
try {
fs.setErasureCodingPolicy(dirPath, ecPolicy);
fs.setErasureCodingPolicy(dirPath, ecPolicy.getName());
fail("FileNotFoundException should be thrown for a non-existent"
+ " file path");
} catch (FileNotFoundException e) {
@ -347,7 +347,7 @@ public void testEcPolicyOnFile() throws Exception {
// Set EC policy on file
try {
fs.setErasureCodingPolicy(ecFilePath, ecPolicy);
fs.setErasureCodingPolicy(ecFilePath, ecPolicy.getName());
fail("IOException should be thrown for setting EC policy on file");
} catch (IOException e) {
assertExceptionContains("Attempt to set an erasure coding policy " +

3
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteReadStripedFile.java
View File

@ -80,7 +80,8 @@ public void setup() throws IOException {
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build();
fs = cluster.getFileSystem();
fs.mkdirs(new Path("/ec"));
cluster.getFileSystem().getClient().setErasureCodingPolicy("/ec", null);
cluster.getFileSystem().getClient().setErasureCodingPolicy("/ec",
ErasureCodingPolicyManager.getSystemDefaultPolicy().getName());
}
@After

3
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteStripedFileWithFailure.java
View File

@ -59,7 +59,8 @@ public class TestWriteStripedFileWithFailure {
public void setup() throws IOException {
conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize);
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build();
cluster.getFileSystem().getClient().setErasureCodingPolicy("/", null);
cluster.getFileSystem().getClient().setErasureCodingPolicy("/",
ErasureCodingPolicyManager.getSystemDefaultPolicy().getName());
fs = cluster.getFileSystem();
}

3
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java
View File

@ -1940,7 +1940,8 @@ private void doTestBalancerWithStripedFile(Configuration conf) throws Exception
cluster.waitActive();
client = NameNodeProxies.createProxy(conf, cluster.getFileSystem(0).getUri(),
ClientProtocol.class).getProxy();
client.setErasureCodingPolicy("/", null);
client.setErasureCodingPolicy("/",
ErasureCodingPolicyManager.getSystemDefaultPolicy().getName());
long totalCapacity = sum(capacities);

4
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFSStriped.java
View File

@ -83,8 +83,8 @@ public void testRead() throws Exception {
.nameNodeHttpPort(ServerSocketUtil.getPort(19870, 100))
.numDataNodes(numDNs)
.build();
cluster.getFileSystem().getClient()
.setErasureCodingPolicy("/", null);
cluster.getFileSystem().getClient().setErasureCodingPolicy("/",
ErasureCodingPolicyManager.getSystemDefaultPolicy().getName());
try {
cluster.waitActive();
doTestRead(conf, cluster, true);

9
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReconstructStripedBlocksWithRackAwareness.java
View File

@ -150,7 +150,8 @@ public void testReconstructForNotEnoughRacks() throws Exception {
.numDataNodes(hosts.length).build();
cluster.waitActive();
fs = cluster.getFileSystem();
fs.setErasureCodingPolicy(new Path("/"), null);
fs.setErasureCodingPolicy(new Path("/"),
ErasureCodingPolicyManager.getSystemDefaultPolicy().getName());
FSNamesystem fsn = cluster.getNamesystem();
BlockManager bm = fsn.getBlockManager();
@ -220,7 +221,8 @@ public void testChooseExcessReplicasToDelete() throws Exception {
.numDataNodes(hosts.length).build();
cluster.waitActive();
fs = cluster.getFileSystem();
fs.setErasureCodingPolicy(new Path("/"), null);
fs.setErasureCodingPolicy(new Path("/"),
ErasureCodingPolicyManager.getSystemDefaultPolicy().getName());
MiniDFSCluster.DataNodeProperties lastHost = stopDataNode(
hosts[hosts.length - 1]);
@ -273,7 +275,8 @@ public void testReconstructionWithDecommission() throws Exception {
.numDataNodes(hostNames.length).build();
cluster.waitActive();
fs = cluster.getFileSystem();
fs.setErasureCodingPolicy(new Path("/"), null);
fs.setErasureCodingPolicy(new Path("/"),
ErasureCodingPolicyManager.getSystemDefaultPolicy().getName());
final BlockManager bm = cluster.getNamesystem().getBlockManager();
final DatanodeManager dm = bm.getDatanodeManager();

4
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestSequentialBlockGroupId.java
View File

@ -88,8 +88,8 @@ public void setup() throws Exception {
blockGrpIdGenerator = cluster.getNamesystem().getBlockManager()
.getBlockIdManager().getBlockGroupIdGenerator();
fs.mkdirs(ecDir);
cluster.getFileSystem().getClient()
.setErasureCodingPolicy("/ecDir", null);
cluster.getFileSystem().getClient().setErasureCodingPolicy("/ecDir",
ErasureCodingPolicyManager.getSystemDefaultPolicy().getName());
}
@After

3
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeErasureCodingMetrics.java
View File

@ -75,7 +75,8 @@ public void setup() throws IOException {
conf.setInt(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY, 1);
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build();
cluster.waitActive();
cluster.getFileSystem().getClient().setErasureCodingPolicy("/", null);
cluster.getFileSystem().getClient().setErasureCodingPolicy("/",
ErasureCodingPolicyManager.getSystemDefaultPolicy().getName());
fs = cluster.getFileSystem();
}

3
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestMover.java
View File

@ -537,7 +537,8 @@ public void testMoverWithStripedFile() throws Exception {
client.setStoragePolicy(barDir,
HdfsConstants.HOT_STORAGE_POLICY_NAME);
// set an EC policy on "/bar" directory
client.setErasureCodingPolicy(barDir, null);
client.setErasureCodingPolicy(barDir,
ErasureCodingPolicyManager.getSystemDefaultPolicy().getName());
// write file to barDir
final String fooFile = "/bar/foo";

3
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddOverReplicatedStripedBlocks.java
View File

@ -81,7 +81,8 @@ public void setup() throws IOException {
cluster.waitActive();
fs = cluster.getFileSystem();
fs.mkdirs(dirPath);
fs.getClient().setErasureCodingPolicy(dirPath.toString(), null);
fs.getClient().setErasureCodingPolicy(dirPath.toString(),
ErasureCodingPolicyManager.getSystemDefaultPolicy().getName());
}
@After

3
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddStripedBlockInFBR.java
View File

@ -87,7 +87,8 @@ public void testAddBlockInFullBlockReport() throws Exception {
final Path repDir = new Path("/rep");
dfs.mkdirs(ecDir);
dfs.mkdirs(repDir);
dfs.getClient().setErasureCodingPolicy(ecDir.toString(), null);
dfs.getClient().setErasureCodingPolicy(ecDir.toString(),
ErasureCodingPolicyManager.getSystemDefaultPolicy().getName());
// create several non-EC files and one EC file
final Path[] repFiles = new Path[groupSize];

3
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddStripedBlocks.java
View File

@ -86,7 +86,8 @@ public void setup() throws IOException {
.numDataNodes(groupSize).build();
cluster.waitActive();
dfs = cluster.getFileSystem();
dfs.getClient().setErasureCodingPolicy("/", null);
dfs.getClient().setErasureCodingPolicy("/", ErasureCodingPolicyManager
.getSystemDefaultPolicy().getName());
}
@After

4
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java
View File

@ -479,7 +479,7 @@ public void testAddNewStripedBlock() throws IOException{
//set the storage policy of the directory
fs.mkdir(new Path(testDir), new FsPermission("755"));
fs.getClient().getNamenode().setErasureCodingPolicy(
testDir, testECPolicy);
testDir, testECPolicy.getName());
// Create a file with striped block
Path p = new Path(testFilePath);
@ -552,7 +552,7 @@ public void testUpdateStripedBlocks() throws IOException{
//set the storage policy of the directory
fs.mkdir(new Path(testDir), new FsPermission("755"));
fs.getClient().getNamenode().setErasureCodingPolicy(
testDir, testECPolicy);
testDir, testECPolicy.getName());
//create a file with striped blocks
Path p = new Path(testFilePath);

6
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java
View File

@ -156,7 +156,7 @@ private void testPersistHelper(Configuration conf) throws IOException {
private void testSaveAndLoadStripedINodeFile(FSNamesystem fsn, Configuration conf,
boolean isUC) throws IOException{
// Construct an INode with StripedBlock for saving and loading
fsn.setErasureCodingPolicy("/", testECPolicy, false);
fsn.setErasureCodingPolicy("/", testECPolicy.getName(), false);
long id = 123456789;
byte[] name = "testSaveAndLoadInodeFile_testfile".getBytes();
PermissionStatus permissionStatus = new PermissionStatus("testuser_a",
@ -472,8 +472,8 @@ public void testSupportBlockGroup() throws Exception {
// Create directories and files
fs.mkdirs(parentDir);
fs.mkdirs(childDir);
fs.setErasureCodingPolicy(parentDir, testECPolicy);
fs.setErasureCodingPolicy(childDir, ec32Policy);
fs.setErasureCodingPolicy(parentDir, testECPolicy.getName());
fs.setErasureCodingPolicy(childDir, ec32Policy.getName());
Path file_10_4 = new Path(parentDir, "striped_file_10_4");
Path file_3_2 = new Path(childDir, "striped_file_3_2");

8
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
View File

@ -690,7 +690,7 @@ public void testFsckOpenECFiles() throws Exception {
util.createFiles(fs, topDir);
// set topDir to EC when it has replicated files
cluster.getFileSystem().getClient().setErasureCodingPolicy(
topDir, ecPolicy);
topDir, ecPolicy.getName());
// create a new file under topDir
DFSTestUtil.createFile(fs, new Path(topDir, "ecFile"), 1024, (short) 1, 0L);
@ -2307,7 +2307,8 @@ public void testFsckCorruptECFile() throws Exception {
// create file
Path ecDirPath = new Path("/striped");
fs.mkdir(ecDirPath, FsPermission.getDirDefault());
fs.getClient().setErasureCodingPolicy(ecDirPath.toString(), null);
fs.getClient().setErasureCodingPolicy(ecDirPath.toString(),
ErasureCodingPolicyManager.getSystemDefaultPolicy().getName());
Path file = new Path(ecDirPath, "corrupted");
final int length = cellSize * dataBlocks;
final byte[] bytes = StripedFileTestUtil.generateBytes(length);
@ -2372,7 +2373,8 @@ public void testFsckMissingECFile() throws Exception {
// create file
Path ecDirPath = new Path("/striped");
fs.mkdir(ecDirPath, FsPermission.getDirDefault());
fs.getClient().setErasureCodingPolicy(ecDirPath.toString(), null);
fs.getClient().setErasureCodingPolicy(ecDirPath.toString(),
ErasureCodingPolicyManager.getSystemDefaultPolicy().getName());
Path file = new Path(ecDirPath, "missing");
final int length = cellSize * dataBlocks;
final byte[] bytes = StripedFileTestUtil.generateBytes(length);

3
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java
View File

@ -738,7 +738,8 @@ public void testVerifyMissingBlockGroupsMetrics() throws Exception {
// create file
Path ecDirPath = new Path("/striped");
fs.mkdir(ecDirPath, FsPermission.getDirDefault());
fs.getClient().setErasureCodingPolicy(ecDirPath.toString(), null);
fs.getClient().setErasureCodingPolicy(ecDirPath.toString(),
ErasureCodingPolicyManager.getSystemDefaultPolicy().getName());
Path file = new Path(ecDirPath, "corrupted");
final int length = cellSize * dataBlocks;
final byte[] bytes = StripedFileTestUtil.generateBytes(length);

3
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestQuotaWithStripedBlocks.java
View File

@ -71,7 +71,8 @@ public void setUp() throws IOException {
dfs = cluster.getFileSystem();
dfs.mkdirs(ecDir);
dfs.getClient().setErasureCodingPolicy(ecDir.toString(), ecPolicy);
dfs.getClient()
.setErasureCodingPolicy(ecDir.toString(), ecPolicy.getName());
dfs.setQuota(ecDir, Long.MAX_VALUE - 1, DISK_QUOTA);
dfs.setQuotaByStorageType(ecDir, StorageType.DISK, DISK_QUOTA);
dfs.setStoragePolicy(ecDir, HdfsConstants.HOT_STORAGE_POLICY_NAME);

6
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestReconstructStripedBlocks.java
View File

@ -201,7 +201,8 @@ public void test2RecoveryTasksForSameBlockGroup() throws Exception {
cluster.waitActive();
DistributedFileSystem fs = cluster.getFileSystem();
BlockManager bm = cluster.getNamesystem().getBlockManager();
fs.getClient().setErasureCodingPolicy("/", null);
fs.getClient().setErasureCodingPolicy("/",
ErasureCodingPolicyManager.getSystemDefaultPolicy().getName());
int fileLen = dataBlocks * blockSize;
Path p = new Path("/test2RecoveryTasksForSameBlockGroup");
final byte[] data = new byte[fileLen];
@ -266,7 +267,8 @@ public void testCountLiveReplicas() throws Exception {
try {
fs.mkdirs(dirPath);
fs.setErasureCodingPolicy(dirPath, null);
fs.setErasureCodingPolicy(dirPath,
ErasureCodingPolicyManager.getSystemDefaultPolicy().getName());
DFSTestUtil.createFile(fs, filePath,
cellSize * dataBlocks * 2, (short) 1, 0L);

6
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStripedINodeFile.java
View File

@ -307,7 +307,8 @@ public void testDeleteOp() throws Exception {
dfs.mkdirs(ecDir);
// set erasure coding policy
dfs.setErasureCodingPolicy(ecDir, null);
dfs.setErasureCodingPolicy(ecDir,
ErasureCodingPolicyManager.getSystemDefaultPolicy().getName());
DFSTestUtil.createFile(dfs, ecFile, len, (short) 1, 0xFEED);
DFSTestUtil.createFile(dfs, contiguousFile, len, (short) 1, 0xFEED);
final FSDirectory fsd = fsn.getFSDirectory();
@ -408,7 +409,8 @@ public void testUnsuitableStoragePoliciesWithECStripedMode()
client.mkdirs(fooDir, new FsPermission((short) 777), true);
client.setStoragePolicy(fooDir, HdfsConstants.ONESSD_STORAGE_POLICY_NAME);
// set an EC policy on "/foo" directory
client.setErasureCodingPolicy(fooDir, null);
client.setErasureCodingPolicy(fooDir,
ErasureCodingPolicyManager.getSystemDefaultPolicy().getName());
// write file to fooDir
final String barFile = "/foo/bar";

3
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java
View File

@ -236,7 +236,8 @@ public static void createOriginalFSImage() throws IOException {
ErasureCodingPolicy ecPolicy =
ErasureCodingPolicyManager.getPolicyByPolicyID(
HdfsConstants.XOR_2_1_POLICY_ID);
hdfs.getClient().setErasureCodingPolicy(ecDir.toString(), ecPolicy);
hdfs.getClient().setErasureCodingPolicy(ecDir.toString(),
ecPolicy.getName());
writtenFiles.put(ecDir.toString(), hdfs.getFileStatus(ecDir));
// Create an empty Erasure Coded file

3
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewerWithStripedBlocks.java
View File

@ -63,7 +63,8 @@ public void setup() throws IOException {
conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize);
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build();
cluster.waitActive();
cluster.getFileSystem().getClient().setErasureCodingPolicy("/", null);
cluster.getFileSystem().getClient().setErasureCodingPolicy("/",
ErasureCodingPolicyManager.getSystemDefaultPolicy().getName());
fs = cluster.getFileSystem();
Path eczone = new Path("/eczone");
fs.mkdirs(eczone);

2
hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml
View File

@ -359,7 +359,7 @@
<comparators>
<comparator>
<type>SubstringComparator</type>
<expected-output>Policy 'invalidpolicy' does not match any of the supported policies. Please select any one of [</expected-output>
<expected-output>Policy 'invalidpolicy' does not match any supported erasure coding policies.</expected-output>
</comparator>
</comparators>
</test>

2
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/TestDFSIO.java
View File

@ -989,7 +989,7 @@ void createAndEnableECOnPath(FileSystem fs, Path path)
((DistributedFileSystem) fs).getAllErasureCodingPolicies();
for (ErasureCodingPolicy ec : list) {
if (erasureCodePolicyName.equals(ec.getName())) {
((DistributedFileSystem) fs).setErasureCodingPolicy(path, ec);
((DistributedFileSystem) fs).setErasureCodingPolicy(path, ec.getName());
LOG.info("enable erasureCodePolicy = " + erasureCodePolicyName +
" on " + path.toString());
break;