big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-11082. Provide replicated EC policy to replicate files. Contributed by SammiChen.

Browse Source
This commit is contained in:
Andrew Wang 2017-08-16 22:17:06 -07:00
parent 08aaa4b36f
commit 96b3a6b972
14 changed files with 331 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ErasureCodeConstants.java
View File

@ -30,6 +30,7 @@ private ErasureCodeConstants() {
public static final String RS_LEGACY_CODEC_NAME = "rs-legacy";
public static final String XOR_CODEC_NAME = "xor";
public static final String HHXOR_CODEC_NAME = "hhxor";
public static final String REPLICATION_CODEC_NAME = "replication";
public static final ECSchema RS_6_3_SCHEMA = new ECSchema(
RS_CODEC_NAME, 6, 3);
@ -45,4 +46,11 @@ private ErasureCodeConstants() {
public static final ECSchema RS_10_4_SCHEMA = new ECSchema(
RS_CODEC_NAME, 10, 4);
public static final ECSchema REPLICATION_1_2_SCHEMA = new ECSchema(
REPLICATION_CODEC_NAME, 1, 2);
public static final byte USER_DEFINED_POLICY_START_ID = (byte) 64;
public static final byte REPLICATION_POLICY_ID = (byte) 63;
public static final String REPLICATION_POLICY_NAME = REPLICATION_CODEC_NAME;
}

3
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
View File

@ -3044,7 +3044,8 @@ void addRetLenToReaderScope(TraceScope scope, int retLen) {
*
* @param src path to get the information for
* @return Returns the policy information if file or directory on the path is
* erasure coded, null otherwise
* erasure coded, null otherwise. Null will be returned if directory or file
* has REPLICATION policy.
* @throws IOException
*/

6
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
View File

@ -2540,7 +2540,8 @@ public Void next(final FileSystem fs, final Path p) throws IOException {
*
* @param path The path of the file or directory
* @return Returns the policy information if file or directory on the path
* is erasure coded, null otherwise
* is erasure coded, null otherwise. Null will be returned if directory or
* file has REPLICATION policy.
* @throws IOException
*/
public ErasureCodingPolicy getErasureCodingPolicy(final Path path)
@ -2567,7 +2568,8 @@ public ErasureCodingPolicy next(final FileSystem fs, final Path p)
}
/**
* Retrieve all the erasure coding policies supported by this file system.
* Retrieve all the erasure coding policies supported by this file system,
* excluding REPLICATION policy.
*
* @return all erasure coding policies supported by this file system.
* @throws IOException

6
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
View File

@ -1588,7 +1588,8 @@ AddECPolicyResponse[] addErasureCodingPolicies(
/**
* Get the erasure coding policies loaded in Namenode.
* Get the erasure coding policies loaded in Namenode, excluding REPLICATION
* policy.
*
* @throws IOException
*/
@ -1604,7 +1605,8 @@ AddECPolicyResponse[] addErasureCodingPolicies(
Map<String, String> getErasureCodingCodecs() throws IOException;
/**
* Get the information about the EC policy for the path.
* Get the information about the EC policy for the path. Null will be returned
* if directory or file has REPLICATION policy.
*
* @param src path to get the info for
* @throws IOException

5
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ErasureCodingPolicy.java
View File

@ -25,6 +25,7 @@
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.io.erasurecode.ECSchema;
import org.apache.hadoop.io.erasurecode.ErasureCodeConstants;
/**
* A policy about how to write/read/code an erasure coding file.
@ -107,6 +108,10 @@ public void setId(byte id) {
this.id = id;
}
public boolean isReplicationPolicy() {
return (id == ErasureCodeConstants.REPLICATION_POLICY_ID);
}
@Override
public boolean equals(Object o) {
if (o == null) {

14
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/SystemErasureCodingPolicies.java
View File

@ -68,6 +68,13 @@ private SystemErasureCodingPolicies() {}
new ErasureCodingPolicy(ErasureCodeConstants.RS_10_4_SCHEMA,
DEFAULT_CELLSIZE, RS_10_4_POLICY_ID);
// REPLICATION policy is always enabled.
private static final ErasureCodingPolicy REPLICATION_POLICY =
new ErasureCodingPolicy(ErasureCodeConstants.REPLICATION_POLICY_NAME,
ErasureCodeConstants.REPLICATION_1_2_SCHEMA,
DEFAULT_CELLSIZE,
ErasureCodeConstants.REPLICATION_POLICY_ID);
private static final List<ErasureCodingPolicy> SYS_POLICIES =
Collections.unmodifiableList(Arrays.asList(
SYS_POLICY1, SYS_POLICY2, SYS_POLICY3, SYS_POLICY4,
@ -118,4 +125,11 @@ public static ErasureCodingPolicy getByID(byte id) {
public static ErasureCodingPolicy getByName(String name) {
return SYSTEM_POLICIES_BY_NAME.get(name);
}
/**
* Get the special REPLICATION policy.
*/
public static ErasureCodingPolicy getReplicationPolicy() {
return REPLICATION_POLICY;
}
}

13
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java
View File

@ -27,6 +27,7 @@
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.io.erasurecode.CodecUtil;
import org.apache.hadoop.io.erasurecode.ErasureCodeConstants;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -48,7 +49,6 @@ public final class ErasureCodingPolicyManager {
public static Logger LOG = LoggerFactory.getLogger(
ErasureCodingPolicyManager.class);
private static final byte USER_DEFINED_POLICY_START_ID = (byte) 64;
private int maxCellSize =
DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_MAX_CELLSIZE_DEFAULT;
@ -157,7 +157,13 @@ public ErasureCodingPolicy[] getEnabledPolicies() {
* Get enabled policy by policy name.
*/
public ErasureCodingPolicy getEnabledPolicyByName(String name) {
return enabledPoliciesByName.get(name);
ErasureCodingPolicy ecPolicy = enabledPoliciesByName.get(name);
if (ecPolicy == null) {
if (name.equalsIgnoreCase(ErasureCodeConstants.REPLICATION_POLICY_NAME)) {
ecPolicy = SystemErasureCodingPolicies.getReplicationPolicy();
}
}
return ecPolicy;
}
/**
@ -257,7 +263,8 @@ public synchronized ErasureCodingPolicy addPolicy(ErasureCodingPolicy policy)
private byte getNextAvailablePolicyID() {
byte currentId = this.userPoliciesByID.keySet().stream()
.max(Byte::compareTo).orElse(USER_DEFINED_POLICY_START_ID);
.max(Byte::compareTo).orElse(
ErasureCodeConstants.USER_DEFINED_POLICY_START_ID);
return (byte) (currentId + 1);
}

13
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java
View File

@ -62,7 +62,7 @@ private FSDirErasureCodingOp() {}
/**
* Check if the ecPolicyName is valid and enabled, return the corresponding
* EC policy if is.
* EC policy if is, including the REPLICATION EC policy.
* @param fsn namespace
* @param ecPolicyName name of EC policy to be checked
* @return an erasure coding policy if ecPolicyName is valid and enabled
@ -295,7 +295,12 @@ static ErasureCodingPolicy getErasureCodingPolicy(final FSNamesystem fsn,
if (iip.getLastINode() == null) {
throw new FileNotFoundException("Path not found: " + iip.getPath());
}
return getErasureCodingPolicyForPath(fsd, iip);
ErasureCodingPolicy ecPolicy = getErasureCodingPolicyForPath(fsd, iip);
if (ecPolicy != null && ecPolicy.isReplicationPolicy()) {
ecPolicy = null;
}
return ecPolicy;
}
/**
@ -312,7 +317,8 @@ static boolean hasErasureCodingPolicy(final FSNamesystem fsn,
}
/**
* Get the erasure coding policy. This does not do any permission checking.
* Get the erasure coding policy, including the REPLICATION policy. This does
* not do any permission checking.
*
* @param fsn namespace
* @param iip inodes in the path containing the file
@ -350,6 +356,7 @@ static Map<String, String> getErasureCodingCodecs(final FSNamesystem fsn)
return CodecRegistry.getInstance().getCodec2CoderCompactMap();
}
//return erasure coding policy for path, including REPLICATION policy
private static ErasureCodingPolicy getErasureCodingPolicyForPath(
FSDirectory fsd, INodesInPath iip) throws IOException {
Preconditions.checkNotNull(iip, "INodes cannot be null");

2
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
View File

@ -541,7 +541,7 @@ private static INodesInPath addFile(
ecPolicy = FSDirErasureCodingOp.unprotectedGetErasureCodingPolicy(
fsd.getFSNamesystem(), existing);
}
if (ecPolicy != null) {
if (ecPolicy != null && (!ecPolicy.isReplicationPolicy())) {
isStriped = true;
}
}

24
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java
View File

@ -25,6 +25,7 @@
import org.apache.hadoop.hdfs.protocol.AddECPolicyResponse;
import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
import org.apache.hadoop.hdfs.util.ECPolicyLoader;
import org.apache.hadoop.io.erasurecode.ErasureCodeConstants;
import org.apache.hadoop.tools.TableListing;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.util.Tool;
@ -309,7 +310,8 @@ public String getName() {
@Override
public String getShortUsage() {
return "[" + getName() + " -path <path> -policy <policy>]\n";
return "[" + getName() +
" -path <path> [-policy <policy>] [-replicate]]\n";
}
@Override
@ -318,9 +320,13 @@ public String getLongUsage() {
listing.addRow("<path>", "The path of the file/directory to set " +
"the erasure coding policy");
listing.addRow("<policy>", "The name of the erasure coding policy");
listing.addRow("-replicate",
"force 3x replication scheme on the directory");
return getShortUsage() + "\n" +
"Set the erasure coding policy for a file/directory.\n\n" +
listing.toString();
listing.toString() + "\n" +
"-replicate and -policy are optional arguments. They cannot been " +
"used at the same time";
}
@Override
@ -332,14 +338,24 @@ public int run(Configuration conf, List<String> args) throws IOException {
return 1;
}
final String ecPolicyName = StringUtils.popOptionWithArgument("-policy",
String ecPolicyName = StringUtils.popOptionWithArgument("-policy",
args);
final boolean replicate = StringUtils.popOption("-replicate", args);
if (args.size() > 0) {
System.err.println(getName() + ": Too many arguments");
return 1;
}
if (replicate) {
if (ecPolicyName != null) {
System.err.println(getName() +
": -replicate and -policy cannot been used at the same time");
return 2;
}
ecPolicyName = ErasureCodeConstants.REPLICATION_POLICY_NAME;
}
final Path p = new Path(path);
final DistributedFileSystem dfs = AdminHelper.getDFS(p.toUri(), conf);
try {
@ -353,7 +369,7 @@ public int run(Configuration conf, List<String> args) throws IOException {
}
} catch (Exception e) {
System.err.println(AdminHelper.prettifyException(e));
return 2;
return 3;
}
return 0;
}

16
hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md
View File

@ -65,9 +65,11 @@ Architecture
2. _The size of a striping cell._ This determines the granularity of striped reads and writes, including buffer sizes and encoding work.
Policies are named *codec*-*num data blocks*-*num parity blocks*-*cell size*. Currently, five built-in policies are supported: `RS-3-2-64k`, `RS-6-3-64k`, `RS-10-4-64k`, `RS-LEGACY-6-3-64k`, and `XOR-2-1-64k`.
Policies are named *codec*-*num data blocks*-*num parity blocks*-*cell size*. Currently, six built-in policies are supported: `RS-3-2-64k`, `RS-6-3-64k`, `RS-10-4-64k`, `RS-LEGACY-6-3-64k`, `XOR-2-1-64k` and `REPLICATION`.
By default, all built-in erasure coding policies are disabled.
`REPLICATION` is a special policy. It can only be set on directory, to force the directory to adopt 3x replication scheme, instead of inheriting its ancestor's erasure coding policy. This policy makes it possible to interleave 3x replication scheme directory with erasure coding directory.
`REPLICATION` policy is always enabled. For other built-in policies, unless they are configured in `dfs.namenode.ec.policies.enabled` property, otherwise they are disabled by default.
Similar to HDFS storage policies, erasure coding policies are set on a directory. When a file is created, it inherits the EC policy of its nearest ancestor directory.
@ -112,7 +114,7 @@ Deployment
what EC policies can be set by clients. It does not affect the behavior of already set file or directory-level EC policies.
By default, all built-in erasure coding policies are disabled. Typically, the cluster administrator will enable set of policies by including them
in the `dfs .namenode.ec.policies.enabled` configuration based on the size of the cluster and the desired fault-tolerance properties. For instance,
in the `dfs.namenode.ec.policies.enabled` configuration based on the size of the cluster and the desired fault-tolerance properties. For instance,
for a cluster with 9 racks, a policy like `RS-10-4-64k` will not preserve rack-level fault-tolerance, and `RS-6-3-64k` or `RS-3-2-64k` might
be more appropriate. If the administrator only cares about node-level fault-tolerance, `RS-10-4-64k` would still be appropriate as long as
there are at least 14 DataNodes in the cluster.
@ -153,7 +155,7 @@ Deployment
HDFS provides an `ec` subcommand to perform administrative commands related to erasure coding.
hdfs ec [generic options]
[-setPolicy -policy <policyName> -path <path>]
[-setPolicy -path <path> [-policy <policyName>] [-replicate]]
[-getPolicy -path <path>]
[-unsetPolicy -path <path>]
[-listPolicies]
@ -165,7 +167,7 @@ Deployment
Below are the details about each command.
* `[-setPolicy -policy <policyName> -path <path>]`
* `[-setPolicy -path <path> [-policy <policyName>] [-replicate]]`
Sets an erasure coding policy on a directory at the specified path.
@ -175,6 +177,10 @@ Below are the details about each command.
This parameter can be omitted if a 'dfs.namenode.ec.system.default.policy' configuration is set.
The EC policy of the path will be set with the default value in configuration.
`-replicate` apply the special `REPLICATION` policy on the directory, force the directory to adopt 3x replication scheme.
`-replicate` and `-policy <policyName>` are optional arguments. They cannot be specified at the same time.
* `[-getPolicy -path <path>]`

81
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java
View File

@ -732,4 +732,85 @@ public Object run() throws Exception {
}
});
}
@Test
public void testReplicationPolicy() throws Exception {
ErasureCodingPolicy replicaPolicy =
SystemErasureCodingPolicies.getReplicationPolicy();
final Path rootDir = new Path("/striped");
final Path replicaDir = new Path(rootDir, "replica");
final Path subReplicaDir = new Path(replicaDir, "replica");
final Path replicaFile = new Path(replicaDir, "file");
final Path subReplicaFile = new Path(subReplicaDir, "file");
fs.mkdirs(rootDir);
fs.setErasureCodingPolicy(rootDir, ecPolicy.getName());
// 1. At first, child directory will inherit parent's EC policy
fs.mkdirs(replicaDir);
fs.createFile(replicaFile).build().close();
HdfsFileStatus fileStatus = (HdfsFileStatus)fs.getFileStatus(replicaFile);
assertEquals("File should inherit EC policy.", ecPolicy, fileStatus
.getErasureCodingPolicy());
assertEquals("File should be a EC file.", true, fileStatus
.isErasureCoded());
assertEquals("File should have the same EC policy as its ancestor.",
ecPolicy, fs.getErasureCodingPolicy(replicaFile));
fs.delete(replicaFile, false);
// 2. Set replication policy on child directory, then get back the policy
fs.setErasureCodingPolicy(replicaDir, replicaPolicy.getName());
ErasureCodingPolicy temp = fs.getErasureCodingPolicy(replicaDir);
assertEquals("Directory should hide replication EC policy.",
null, temp);
// 3. New file will be replication file. Please be noted that replication
// policy only set on directory, not on file
fs.createFile(replicaFile).build().close();
assertEquals("Replication file should have default replication factor.",
fs.getDefaultReplication(),
fs.getFileStatus(replicaFile).getReplication());
fs.setReplication(replicaFile, (short) 2);
assertEquals("File should have replication factor as expected.",
2, fs.getFileStatus(replicaFile).getReplication());
fileStatus = (HdfsFileStatus)fs.getFileStatus(replicaFile);
assertEquals("File should not have EC policy.", null, fileStatus
.getErasureCodingPolicy());
assertEquals("File should not be a EC file.", false,
fileStatus.isErasureCoded());
ErasureCodingPolicy ecPolicyOnFile = fs.getErasureCodingPolicy(replicaFile);
assertEquals("File should not have EC policy.", null, ecPolicyOnFile);
fs.delete(replicaFile, false);
// 4. New directory under replication directory, is also replication
// directory
fs.mkdirs(subReplicaDir);
assertEquals("Directory should inherit hiding replication EC policy.",
null, fs.getErasureCodingPolicy(subReplicaDir));
fs.createFile(subReplicaFile).build().close();
assertEquals("File should have default replication factor.",
fs.getDefaultReplication(),
fs.getFileStatus(subReplicaFile).getReplication());
fileStatus = (HdfsFileStatus)fs.getFileStatus(subReplicaFile);
assertEquals("File should not have EC policy.", null,
fileStatus.getErasureCodingPolicy());
assertEquals("File should not be a EC file.", false,
fileStatus.isErasureCoded());
assertEquals("File should not have EC policy.", null,
fs.getErasureCodingPolicy(subReplicaFile));
fs.delete(subReplicaFile, false);
// 5. Unset replication policy on directory, new file will be EC file
fs.unsetErasureCodingPolicy(replicaDir);
fs.createFile(subReplicaFile).build().close();
fileStatus = (HdfsFileStatus)fs.getFileStatus(subReplicaFile);
assertEquals("File should inherit EC policy.", ecPolicy,
fileStatus.getErasureCodingPolicy());
assertEquals("File should be a EC file.", true,
fileStatus.isErasureCoded());
assertEquals("File should have the same EC policy as its ancestor",
ecPolicy, fs.getErasureCodingPolicy(subReplicaFile));
fs.delete(subReplicaFile, false);
}
}

87
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java
View File

@ -723,4 +723,91 @@ public void testBlockTypeProtoDefaultsToContiguous() throws Exception {
.getBlockType());
assertEquals(defaultBlockType, BlockType.CONTIGUOUS);
}
/**
* Test if a INodeFile under a replication EC policy directory
* can be saved by FSImageSerialization and loaded by FSImageFormat#Loader.
*/
@Test
public void testSaveAndLoadFileUnderReplicationPolicyDir()
throws IOException {
Configuration conf = new Configuration();
DFSTestUtil.enableAllECPolicies(conf);
MiniDFSCluster cluster = null;
try {
cluster = new MiniDFSCluster.Builder(conf).build();
cluster.waitActive();
FSNamesystem fsn = cluster.getNamesystem();
DistributedFileSystem fs = cluster.getFileSystem();
ErasureCodingPolicy replicaPolicy =
SystemErasureCodingPolicies.getReplicationPolicy();
ErasureCodingPolicy defaultEcPolicy =
StripedFileTestUtil.getDefaultECPolicy();
final Path ecDir = new Path("/ec");
final Path replicaDir = new Path(ecDir, "replica");
final Path replicaFile1 = new Path(replicaDir, "f1");
final Path replicaFile2 = new Path(replicaDir, "f2");
// create root directory
fs.mkdir(ecDir, null);
fs.setErasureCodingPolicy(ecDir, defaultEcPolicy.getName());
// create directory, and set replication Policy
fs.mkdir(replicaDir, null);
fs.setErasureCodingPolicy(replicaDir, replicaPolicy.getName());
// create an empty file f1
fs.create(replicaFile1).close();
// create an under-construction file f2
FSDataOutputStream out = fs.create(replicaFile2, (short) 2);
out.writeBytes("hello");
((DFSOutputStream) out.getWrappedStream()).hsync(EnumSet
.of(SyncFlag.UPDATE_LENGTH));
// checkpoint
fs.setSafeMode(SafeModeAction.SAFEMODE_ENTER);
fs.saveNamespace();
fs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE);
cluster.restartNameNode();
cluster.waitActive();
fs = cluster.getFileSystem();
assertTrue(fs.getFileStatus(ecDir).isDirectory());
assertTrue(fs.getFileStatus(replicaDir).isDirectory());
assertTrue(fs.exists(replicaFile1));
assertTrue(fs.exists(replicaFile2));
// check directories
assertEquals("Directory should have default EC policy.",
defaultEcPolicy, fs.getErasureCodingPolicy(ecDir));
assertEquals("Directory should hide replication EC policy.",
null, fs.getErasureCodingPolicy(replicaDir));
// check file1
assertEquals("File should not have EC policy.", null,
fs.getErasureCodingPolicy(replicaFile1));
// check internals of file2
INodeFile file2Node =
fsn.dir.getINode4Write(replicaFile2.toString()).asFile();
assertEquals("hello".length(), file2Node.computeFileSize());
assertTrue(file2Node.isUnderConstruction());
BlockInfo[] blks = file2Node.getBlocks();
assertEquals(1, blks.length);
assertEquals(BlockUCState.UNDER_CONSTRUCTION, blks[0].getBlockUCState());
assertEquals("File should return expected replication factor.",
2, blks[0].getReplication());
assertEquals("File should not have EC policy.", null,
fs.getErasureCodingPolicy(replicaFile2));
// check lease manager
Lease lease = fsn.leaseManager.getLease(file2Node);
Assert.assertNotNull(lease);
} finally {
if (cluster != null) {
cluster.shutdown();
}
}
}
}

78
hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml
View File

@ -101,7 +101,7 @@
</comparator>
<comparator>
<type>SubstringComparator</type>
<expected-output>[-setPolicy -path &lt;path&gt; -policy &lt;policy&gt;]</expected-output>
<expected-output>[-setPolicy -path &lt;path&gt; [-policy &lt;policy&gt;] [-replicate]]</expected-output>
</comparator>
</comparators>
</test>
@ -237,6 +237,29 @@
</comparators>
</test>
<test>
<description>setPolicy : set replication policy on a directory</description>
<test-commands>
<command>-fs NAMENODE -mkdir /ecdir</command>
<ec-admin-command>-fs NAMENODE -setPolicy -policy RS-6-3-64k -path /ecdir</ec-admin-command>
<command>-fs NAMENODE -mkdir /ecdir/replica</command>
<ec-admin-command>-fs NAMENODE -setPolicy -replicate -path /ecdir/replica</ec-admin-command>
<command>-fs NAMENODE -touchz /ecdir/replica/file</command>
<ec-admin-command>-fs NAMENODE -getPolicy -path /ecdir/replica/file</ec-admin-command>
</test-commands>
<cleanup-commands>
<command>-fs NAMENODE -rm /ecdir/replica/file</command>
<command>-fs NAMENODE -rmdir /ecdir/replica</command>
<command>-fs NAMENODE -rmdir /ecdir</command>
</cleanup-commands>
<comparators>
<comparator>
<type>SubstringComparator</type>
<expected-output>is unspecified</expected-output>
</comparator>
</comparators>
</test>
<test>
<description>unsetPolicy : unset policy and get</description>
<test-commands>
@ -453,7 +476,7 @@
<!-- Test illegal parameters -->
<test>
<description>setPolicy : illegal parameters - path is missing</description>
<description>setPolicy : illegal parameters - path option is missing</description>
<test-commands>
<command>-fs NAMENODE -mkdir /ecdir</command>
<ec-admin-command>-fs NAMENODE -setPolicy</ec-admin-command>
@ -470,7 +493,7 @@
</test>
<test>
<description>setPolicy : illegal parameters - policy name is missing</description>
<description>setPolicy : illegal parameters - path name is missing</description>
<test-commands>
<command>-fs NAMENODE -mkdir /ecdir</command>
<ec-admin-command>-fs NAMENODE -setPolicy -path</ec-admin-command>
@ -487,7 +510,7 @@
</test>
<test>
<description>setPolicy : illegal parameters - too many arguments</description>
<description>setPolicy : illegal parameters - too many arguments case 1</description>
<test-commands>
<command>-fs NAMENODE -mkdir /ecdir</command>
<ec-admin-command>-fs NAMENODE -setPolicy -path /ecdir1 -policy RS-3-2-64k /ecdir2</ec-admin-command>
@ -503,6 +526,23 @@
</comparators>
</test>
<test>
<description>setPolicy : illegal parameters - too many arguments case 2</description>
<test-commands>
<command>-fs NAMENODE -mkdir /ecdir</command>
<ec-admin-command>-fs NAMENODE -setPolicy -path /ecdir1 -policy RS-3-2-64k -replicate /ecdir2</ec-admin-command>
</test-commands>
<cleanup-commands>
<command>-fs NAMENODE -rmdir /ecdir</command>
</cleanup-commands>
<comparators>
<comparator>
<type>SubstringComparator</type>
<expected-output>-setPolicy: Too many arguments</expected-output>
</comparator>
</comparators>
</test>
<test>
<description>setPolicy : illegal parameters - invalidpolicy</description>
<test-commands>
@ -552,6 +592,36 @@
</comparators>
</test>
<test>
<description>setPolicy : illegal parameters - wrong spelling replicate </description>
<test-commands>
<ec-admin-command>-fs NAMENODE -setPolicy -path /ecdir -replica</ec-admin-command>
</test-commands>
<cleanup-commands>
</cleanup-commands>
<comparators>
<comparator>
<type>SubstringComparator</type>
<expected-output>-setPolicy: Too many arguments</expected-output>
</comparator>
</comparators>
</test>
<test>
<description>setPolicy : illegal parameters - replicate and policy coexist</description>
<test-commands>
<ec-admin-command>-fs NAMENODE -setPolicy -path /ecdir -policy RS-3-2-64k -replicate</ec-admin-command>
</test-commands>
<cleanup-commands>
</cleanup-commands>
<comparators>
<comparator>
<type>SubstringComparator</type>
<expected-output>-replicate and -policy cannot been used at the same time</expected-output>
</comparator>
</comparators>
</test>
<test>
<description>setPolicy : set erasure coding policy without given a specific policy name</description>
<test-commands>