big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-10706. [READ] Add tool generating FSImage from external store

Browse Source
This commit is contained in:
Virajith Jalaparti 2017-04-15 12:15:08 -07:00 committed by Chris Douglas
parent b668eb9155
commit 8da3a6e314
21 changed files with 2438 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

87
hadoop-tools/hadoop-fs2img/pom.xml Normal file
View File

@ -0,0 +1,87 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. See accompanying LICENSE file.
-->
<project>
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
<version>3.0.0-alpha3-SNAPSHOT</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-fs2img</artifactId>
<version>3.0.0-alpha3-SNAPSHOT</version>
<description>fs2img</description>
<name>fs2img</name>
<packaging>jar</packaging>
<properties>
<hadoop.log.dir>${project.build.directory}/log</hadoop.log.dir>
</properties>
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-minicluster</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>commons-cli</groupId>
<artifactId>commons-cli</artifactId>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-all</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
<configuration>
<archive>
<manifest>
<mainClass>org.apache.hadoop.hdfs.server.namenode.FileSystemImage</mainClass>
</manifest>
</archive>
</configuration>
</plugin>
</plugins>
</build>
</project>

95
hadoop-tools/hadoop-fs2img/src/main/java/org/apache/hadoop/hdfs/server/namenode/BlockResolver.java Normal file
View File

@ -0,0 +1,95 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdfs.server.namenode;
import java.util.ArrayList;
import java.util.List;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.BlockProto;
/**
* Given an external reference, create a sequence of blocks and associated
* metadata.
*/
public abstract class BlockResolver {
protected BlockProto buildBlock(long blockId, long bytes) {
return buildBlock(blockId, bytes, 1001);
}
protected BlockProto buildBlock(long blockId, long bytes, long genstamp) {
BlockProto.Builder b = BlockProto.newBuilder()
.setBlockId(blockId)
.setNumBytes(bytes)
.setGenStamp(genstamp);
return b.build();
}
/**
* @param s the external reference.
* @return sequence of blocks that make up the reference.
*/
public Iterable<BlockProto> resolve(FileStatus s) {
List<Long> lengths = blockLengths(s);
ArrayList<BlockProto> ret = new ArrayList<>(lengths.size());
long tot = 0;
for (long l : lengths) {
tot += l;
ret.add(buildBlock(nextId(), l));
}
if (tot != s.getLen()) {
// log a warning?
throw new IllegalStateException(
"Expected " + s.getLen() + " found " + tot);
}
return ret;
}
/**
* @return the next block id.
*/
public abstract long nextId();
/**
* @return the maximum sequentially allocated block ID for this filesystem.
*/
protected abstract long lastId();
/**
* @param status the external reference.
* @return the lengths of the resultant blocks.
*/
protected abstract List<Long> blockLengths(FileStatus status);
/**
* @param status the external reference.
* @return the block size to assign to this external reference.
*/
public long preferredBlockSize(FileStatus status) {
return status.getBlockSize();
}
/**
* @param status the external reference.
* @return the replication to assign to this external reference.
*/
public abstract int getReplication(FileStatus status);
}

105
hadoop-tools/hadoop-fs2img/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSTreeWalk.java Normal file
View File

@ -0,0 +1,105 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdfs.server.namenode;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.ConcurrentModificationException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
/**
* Traversal of an external FileSystem.
*/
public class FSTreeWalk extends TreeWalk {
private final Path root;
private final FileSystem fs;
public FSTreeWalk(Path root, Configuration conf) throws IOException {
this.root = root;
fs = root.getFileSystem(conf);
}
@Override
protected Iterable<TreePath> getChildren(TreePath path, long id,
TreeIterator i) {
// TODO symlinks
if (!path.getFileStatus().isDirectory()) {
return Collections.emptyList();
}
try {
ArrayList<TreePath> ret = new ArrayList<>();
for (FileStatus s : fs.listStatus(path.getFileStatus().getPath())) {
ret.add(new TreePath(s, id, i));
}
return ret;
} catch (FileNotFoundException e) {
throw new ConcurrentModificationException("FS modified");
} catch (IOException e) {
throw new RuntimeException(e);
}
}
class FSTreeIterator extends TreeIterator {
private FSTreeIterator() {
}
FSTreeIterator(TreePath p) {
getPendingQueue().addFirst(
new TreePath(p.getFileStatus(), p.getParentId(), this));
}
FSTreeIterator(Path p) throws IOException {
try {
FileStatus s = fs.getFileStatus(root);
getPendingQueue().addFirst(new TreePath(s, -1L, this));
} catch (FileNotFoundException e) {
if (p.equals(root)) {
throw e;
}
throw new ConcurrentModificationException("FS modified");
}
}
@Override
public TreeIterator fork() {
if (getPendingQueue().isEmpty()) {
return new FSTreeIterator();
}
return new FSTreeIterator(getPendingQueue().removeFirst());
}
}
@Override
public TreeIterator iterator() {
try {
return new FSTreeIterator(root);
} catch (IOException e) {
throw new RuntimeException(e);
}
}
}

139
hadoop-tools/hadoop-fs2img/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileSystemImage.java Normal file
View File

@ -0,0 +1,139 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdfs.server.namenode;
import java.io.File;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
import org.apache.commons.cli.PosixParser;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.server.common.BlockFormat;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
/**
* Create FSImage from an external namespace.
*/
public class FileSystemImage implements Tool {
private Configuration conf;
@Override
public Configuration getConf() {
return conf;
}
@Override
public void setConf(Configuration conf) {
this.conf = conf;
// require absolute URI to write anywhere but local
FileSystem.setDefaultUri(conf, new File(".").toURI().toString());
}
protected void printUsage() {
HelpFormatter formatter = new HelpFormatter();
formatter.printHelp("fs2img [OPTIONS] URI", new Options());
formatter.setSyntaxPrefix("");
formatter.printHelp("Options", options());
ToolRunner.printGenericCommandUsage(System.out);
}
static Options options() {
Options options = new Options();
options.addOption("o", "outdir", true, "Output directory");
options.addOption("u", "ugiclass", true, "UGI resolver class");
options.addOption("b", "blockclass", true, "Block output class");
options.addOption("i", "blockidclass", true, "Block resolver class");
options.addOption("c", "cachedirs", true, "Max active dirents");
options.addOption("h", "help", false, "Print usage");
return options;
}
@Override
public int run(String[] argv) throws Exception {
Options options = options();
CommandLineParser parser = new PosixParser();
CommandLine cmd;
try {
cmd = parser.parse(options, argv);
} catch (ParseException e) {
System.out.println(
"Error parsing command-line options: " + e.getMessage());
printUsage();
return -1;
}
if (cmd.hasOption("h")) {
printUsage();
return -1;
}
ImageWriter.Options opts =
ReflectionUtils.newInstance(ImageWriter.Options.class, getConf());
for (Option o : cmd.getOptions()) {
switch (o.getOpt()) {
case "o":
opts.output(o.getValue());
break;
case "u":
opts.ugi(Class.forName(o.getValue()).asSubclass(UGIResolver.class));
break;
case "b":
opts.blocks(
Class.forName(o.getValue()).asSubclass(BlockFormat.class));
break;
case "i":
opts.blockIds(
Class.forName(o.getValue()).asSubclass(BlockResolver.class));
break;
case "c":
opts.cache(Integer.parseInt(o.getValue()));
break;
default:
throw new UnsupportedOperationException("Internal error");
}
}
String[] rem = cmd.getArgs();
if (rem.length != 1) {
printUsage();
return -1;
}
try (ImageWriter w = new ImageWriter(opts)) {
for (TreePath e : new FSTreeWalk(new Path(rem[0]), getConf())) {
w.accept(e); // add and continue
}
}
return 0;
}
public static void main(String[] argv) throws Exception {
int ret = ToolRunner.run(new FileSystemImage(), argv);
System.exit(ret);
}
}

44
hadoop-tools/hadoop-fs2img/src/main/java/org/apache/hadoop/hdfs/server/namenode/FixedBlockMultiReplicaResolver.java Normal file
View File

@ -0,0 +1,44 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdfs.server.namenode;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
/**
* Resolver mapping all files to a configurable, uniform blocksize
* and replication.
*/
public class FixedBlockMultiReplicaResolver extends FixedBlockResolver {
public static final String REPLICATION =
"hdfs.image.writer.resolver.fixed.block.replication";
private int replication;
@Override
public void setConf(Configuration conf) {
super.setConf(conf);
replication = conf.getInt(REPLICATION, 1);
}
public int getReplication(FileStatus s) {
return replication;
}
}

93
hadoop-tools/hadoop-fs2img/src/main/java/org/apache/hadoop/hdfs/server/namenode/FixedBlockResolver.java Normal file
View File

@ -0,0 +1,93 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdfs.server.namenode;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.hadoop.conf.Configurable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
/**
* Resolver mapping all files to a configurable, uniform blocksize.
*/
public class FixedBlockResolver extends BlockResolver implements Configurable {
public static final String BLOCKSIZE =
"hdfs.image.writer.resolver.fixed.block.size";
public static final String START_BLOCK =
"hdfs.image.writer.resolver.fixed.block.start";
private Configuration conf;
private long blocksize = 256 * (1L << 20);
private final AtomicLong blockIds = new AtomicLong(0);
@Override
public void setConf(Configuration conf) {
this.conf = conf;
blocksize = conf.getLong(BLOCKSIZE, 256 * (1L << 20));
blockIds.set(conf.getLong(START_BLOCK, (1L << 30)));
}
@Override
public Configuration getConf() {
return conf;
}
@Override
protected List<Long> blockLengths(FileStatus s) {
ArrayList<Long> ret = new ArrayList<>();
if (!s.isFile()) {
return ret;
}
if (0 == s.getLen()) {
// the file has length 0; so we will have one block of size 0
ret.add(0L);
return ret;
}
int nblocks = (int)((s.getLen() - 1) / blocksize) + 1;
for (int i = 0; i < nblocks - 1; ++i) {
ret.add(blocksize);
}
long rem = s.getLen() % blocksize;
ret.add(0 == (rem % blocksize) ? blocksize : rem);
return ret;
}
@Override
public long nextId() {
return blockIds.incrementAndGet();
}
@Override
public long lastId() {
return blockIds.get();
}
@Override
public long preferredBlockSize(FileStatus s) {
return blocksize;
}
@Override
public int getReplication(FileStatus s) {
return 1;
}
}

58
hadoop-tools/hadoop-fs2img/src/main/java/org/apache/hadoop/hdfs/server/namenode/FsUGIResolver.java Normal file
View File

@ -0,0 +1,58 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdfs.server.namenode;
import java.util.HashSet;
import java.util.Set;
/**
* Dynamically assign ids to users/groups as they appear in the external
* filesystem.
*/
public class FsUGIResolver extends UGIResolver {
private int id;
private final Set<String> usernames;
private final Set<String> groupnames;
FsUGIResolver() {
super();
id = 0;
usernames = new HashSet<String>();
groupnames = new HashSet<String>();
}
@Override
public synchronized void addUser(String name) {
if (!usernames.contains(name)) {
addUser(name, id);
id++;
usernames.add(name);
}
}
@Override
public synchronized void addGroup(String name) {
if (!groupnames.contains(name)) {
addGroup(name, id);
id++;
groupnames.add(name);
}
}
}

600
hadoop-tools/hadoop-fs2img/src/main/java/org/apache/hadoop/hdfs/server/namenode/ImageWriter.java Normal file
View File

@ -0,0 +1,600 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdfs.server.namenode;
import java.io.BufferedOutputStream;
import java.io.Closeable;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.FilterOutputStream;
import java.io.OutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.security.DigestOutputStream;
import java.security.MessageDigest;
import java.util.Arrays;
import java.util.Collections;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.Map.Entry;
import java.util.concurrent.atomic.AtomicLong;
import com.google.common.base.Charsets;
import com.google.protobuf.CodedOutputStream;
import org.apache.hadoop.conf.Configurable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocalFileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.server.common.BlockFormat;
import org.apache.hadoop.hdfs.server.common.FileRegion;
import org.apache.hadoop.hdfs.server.namenode.FSImageFormatProtobuf.SectionName;
import org.apache.hadoop.hdfs.server.namenode.FsImageProto.CacheManagerSection;
import org.apache.hadoop.hdfs.server.namenode.FsImageProto.FileSummary;
import org.apache.hadoop.hdfs.server.namenode.FsImageProto.FilesUnderConstructionSection;
import org.apache.hadoop.hdfs.server.namenode.FsImageProto.INodeDirectorySection.DirEntry;
import org.apache.hadoop.hdfs.server.namenode.FsImageProto.INodeSection.INode;
import org.apache.hadoop.hdfs.server.namenode.FsImageProto.INodeSection;
import org.apache.hadoop.hdfs.server.namenode.FsImageProto.NameSystemSection;
import org.apache.hadoop.hdfs.server.namenode.FsImageProto.SecretManagerSection;
import org.apache.hadoop.hdfs.server.namenode.FsImageProto.SnapshotDiffSection;
import org.apache.hadoop.hdfs.server.namenode.FsImageProto.StringTableSection;
import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.MD5Hash;
import org.apache.hadoop.io.compress.CompressionCodec;
import org.apache.hadoop.io.compress.CompressorStream;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.util.StringUtils;
import static org.apache.hadoop.hdfs.server.namenode.FSImageUtil.MAGIC_HEADER;
/**
* Utility crawling an existing hierarchical FileSystem and emitting
* a valid FSImage/NN storage.
*/
// TODO: generalize to types beyond FileRegion
public class ImageWriter implements Closeable {
private static final int ONDISK_VERSION = 1;
private static final int LAYOUT_VERSION = -64; // see NameNodeLayoutVersion
private final Path outdir;
private final FileSystem outfs;
private final File dirsTmp;
private final OutputStream dirs;
private final File inodesTmp;
private final OutputStream inodes;
private final MessageDigest digest;
private final FSImageCompression compress;
private final long startBlock;
private final long startInode;
private final UGIResolver ugis;
private final BlockFormat.Writer<FileRegion> blocks;
private final BlockResolver blockIds;
private final Map<Long, DirEntry.Builder> dircache;
private final TrackedOutputStream<DigestOutputStream> raw;
private boolean closed = false;
private long curSec;
private long curBlock;
private final AtomicLong curInode;
private final FileSummary.Builder summary = FileSummary.newBuilder()
.setOndiskVersion(ONDISK_VERSION)
.setLayoutVersion(LAYOUT_VERSION);
private final String blockPoolID;
public static Options defaults() {
return new Options();
}
@SuppressWarnings("unchecked")
public ImageWriter(Options opts) throws IOException {
final OutputStream out;
if (null == opts.outStream) {
FileSystem fs = opts.outdir.getFileSystem(opts.getConf());
outfs = (fs instanceof LocalFileSystem)
? ((LocalFileSystem)fs).getRaw()
: fs;
Path tmp = opts.outdir;
if (!outfs.mkdirs(tmp)) {
throw new IOException("Failed to create output dir: " + tmp);
}
try (NNStorage stor = new NNStorage(opts.getConf(),
Arrays.asList(tmp.toUri()), Arrays.asList(tmp.toUri()))) {
NamespaceInfo info = NNStorage.newNamespaceInfo();
if (info.getLayoutVersion() != LAYOUT_VERSION) {
throw new IllegalStateException("Incompatible layout " +
info.getLayoutVersion() + " (expected " + LAYOUT_VERSION);
}
stor.format(info);
blockPoolID = info.getBlockPoolID();
}
outdir = new Path(tmp, "current");
out = outfs.create(new Path(outdir, "fsimage_0000000000000000000"));
} else {
// XXX necessary? writing a NNStorage now...
outdir = null;
outfs = null;
out = opts.outStream;
blockPoolID = "";
}
digest = MD5Hash.getDigester();
raw = new TrackedOutputStream<>(new DigestOutputStream(
new BufferedOutputStream(out), digest));
compress = opts.compress;
CompressionCodec codec = compress.getImageCodec();
if (codec != null) {
summary.setCodec(codec.getClass().getCanonicalName());
}
startBlock = opts.startBlock;
curBlock = startBlock;
startInode = opts.startInode;
curInode = new AtomicLong(startInode);
dircache = Collections.synchronizedMap(new DirEntryCache(opts.maxdircache));
ugis = null == opts.ugis
? ReflectionUtils.newInstance(opts.ugisClass, opts.getConf())
: opts.ugis;
BlockFormat<FileRegion> fmt = null == opts.blocks
? ReflectionUtils.newInstance(opts.blockFormatClass, opts.getConf())
: opts.blocks;
blocks = fmt.getWriter(null);
blockIds = null == opts.blockIds
? ReflectionUtils.newInstance(opts.blockIdsClass, opts.getConf())
: opts.blockIds;
// create directory and inode sections as side-files.
// The details are written to files to avoid keeping them in memory.
dirsTmp = File.createTempFile("fsimg_dir", null);
dirsTmp.deleteOnExit();
dirs = beginSection(new FileOutputStream(dirsTmp));
try {
inodesTmp = File.createTempFile("fsimg_inode", null);
inodesTmp.deleteOnExit();
inodes = new FileOutputStream(inodesTmp);
} catch (IOException e) {
// appropriate to close raw?
IOUtils.cleanup(null, raw, dirs);
throw e;
}
raw.write(MAGIC_HEADER);
curSec = raw.pos;
assert raw.pos == MAGIC_HEADER.length;
}
public void accept(TreePath e) throws IOException {
assert e.getParentId() < curInode.get();
// allocate ID
long id = curInode.getAndIncrement();
e.accept(id);
assert e.getId() < curInode.get();
INode n = e.toINode(ugis, blockIds, blocks, blockPoolID);
writeInode(n);
if (e.getParentId() > 0) {
// add DirEntry to map, which may page out entries
DirEntry.Builder de = DirEntry.newBuilder()
.setParent(e.getParentId())
.addChildren(e.getId());
dircache.put(e.getParentId(), de);
}
}
@SuppressWarnings("serial")
class DirEntryCache extends LinkedHashMap<Long, DirEntry.Builder> {
// should cache path to root, not evict LRCached
private final int nEntries;
DirEntryCache(int nEntries) {
this.nEntries = nEntries;
}
@Override
public DirEntry.Builder put(Long p, DirEntry.Builder b) {
DirEntry.Builder e = get(p);
if (null == e) {
return super.put(p, b);
}
//merge
e.addAllChildren(b.getChildrenList());
// not strictly conforming
return e;
}
@Override
protected boolean removeEldestEntry(Entry<Long, DirEntry.Builder> be) {
if (size() > nEntries) {
DirEntry d = be.getValue().build();
try {
writeDirEntry(d);
} catch (IOException e) {
throw new RuntimeException(e);
}
return true;
}
return false;
}
}
synchronized void writeInode(INode n) throws IOException {
n.writeDelimitedTo(inodes);
}
synchronized void writeDirEntry(DirEntry e) throws IOException {
e.writeDelimitedTo(dirs);
}
// from FSImageFormatProtobuf... why not just read position from the stream?
private static int getOndiskSize(com.google.protobuf.GeneratedMessage s) {
return CodedOutputStream.computeRawVarint32Size(s.getSerializedSize())
+ s.getSerializedSize();
}
@Override
public synchronized void close() throws IOException {
if (closed) {
return;
}
for (DirEntry.Builder b : dircache.values()) {
DirEntry e = b.build();
writeDirEntry(e);
}
dircache.clear();
// close side files
IOUtils.cleanup(null, dirs, inodes, blocks);
if (null == dirs || null == inodes) {
// init failed
if (raw != null) {
raw.close();
}
return;
}
try {
writeNameSystemSection();
writeINodeSection();
writeDirSection();
writeStringTableSection();
// write summary directly to raw
FileSummary s = summary.build();
s.writeDelimitedTo(raw);
int length = getOndiskSize(s);
byte[] lengthBytes = new byte[4];
ByteBuffer.wrap(lengthBytes).asIntBuffer().put(length);
raw.write(lengthBytes);
} finally {
raw.close();
}
writeMD5("fsimage_0000000000000000000");
closed = true;
}
/**
* Write checksum for image file. Pulled from MD5Utils/internals. Awkward to
* reuse existing tools/utils.
*/
void writeMD5(String imagename) throws IOException {
if (null == outdir) {
//LOG.warn("Not writing MD5");
return;
}
MD5Hash md5 = new MD5Hash(digest.digest());
String digestString = StringUtils.byteToHexString(md5.getDigest());
Path chk = new Path(outdir, imagename + ".md5");
try (OutputStream out = outfs.create(chk)) {
String md5Line = digestString + " *" + imagename + "\n";
out.write(md5Line.getBytes(Charsets.UTF_8));
}
}
OutputStream beginSection(OutputStream out) throws IOException {
CompressionCodec codec = compress.getImageCodec();
if (null == codec) {
return out;
}
return codec.createOutputStream(out);
}
void endSection(OutputStream out, SectionName name) throws IOException {
CompressionCodec codec = compress.getImageCodec();
if (codec != null) {
((CompressorStream)out).finish();
}
out.flush();
long length = raw.pos - curSec;
summary.addSections(FileSummary.Section.newBuilder()
.setName(name.toString()) // not strictly correct, but name not visible
.setOffset(curSec).setLength(length));
curSec += length;
}
void writeNameSystemSection() throws IOException {
NameSystemSection.Builder b = NameSystemSection.newBuilder()
.setGenstampV1(1000)
.setGenstampV1Limit(0)
.setGenstampV2(1001)
.setLastAllocatedBlockId(blockIds.lastId())
.setTransactionId(0);
NameSystemSection s = b.build();
OutputStream sec = beginSection(raw);
s.writeDelimitedTo(sec);
endSection(sec, SectionName.NS_INFO);
}
void writeINodeSection() throws IOException {
// could reset dict to avoid compression cost in close
INodeSection.Builder b = INodeSection.newBuilder()
.setNumInodes(curInode.get() - startInode)
.setLastInodeId(curInode.get());
INodeSection s = b.build();
OutputStream sec = beginSection(raw);
s.writeDelimitedTo(sec);
// copy inodes
try (FileInputStream in = new FileInputStream(inodesTmp)) {
IOUtils.copyBytes(in, sec, 4096, false);
}
endSection(sec, SectionName.INODE);
}
void writeDirSection() throws IOException {
// No header, so dirs can be written/compressed independently
//INodeDirectorySection.Builder b = INodeDirectorySection.newBuilder();
OutputStream sec = raw;
// copy dirs
try (FileInputStream in = new FileInputStream(dirsTmp)) {
IOUtils.copyBytes(in, sec, 4096, false);
}
endSection(sec, SectionName.INODE_DIR);
}
void writeFilesUCSection() throws IOException {
FilesUnderConstructionSection.Builder b =
FilesUnderConstructionSection.newBuilder();
FilesUnderConstructionSection s = b.build();
OutputStream sec = beginSection(raw);
s.writeDelimitedTo(sec);
endSection(sec, SectionName.FILES_UNDERCONSTRUCTION);
}
void writeSnapshotDiffSection() throws IOException {
SnapshotDiffSection.Builder b = SnapshotDiffSection.newBuilder();
SnapshotDiffSection s = b.build();
OutputStream sec = beginSection(raw);
s.writeDelimitedTo(sec);
endSection(sec, SectionName.SNAPSHOT_DIFF);
}
void writeSecretManagerSection() throws IOException {
SecretManagerSection.Builder b = SecretManagerSection.newBuilder()
.setCurrentId(0)
.setTokenSequenceNumber(0);
SecretManagerSection s = b.build();
OutputStream sec = beginSection(raw);
s.writeDelimitedTo(sec);
endSection(sec, SectionName.SECRET_MANAGER);
}
void writeCacheManagerSection() throws IOException {
CacheManagerSection.Builder b = CacheManagerSection.newBuilder()
.setNumPools(0)
.setNumDirectives(0)
.setNextDirectiveId(1);
CacheManagerSection s = b.build();
OutputStream sec = beginSection(raw);
s.writeDelimitedTo(sec);
endSection(sec, SectionName.CACHE_MANAGER);
}
void writeStringTableSection() throws IOException {
StringTableSection.Builder b = StringTableSection.newBuilder();
Map<Integer, String> u = ugis.ugiMap();
b.setNumEntry(u.size());
StringTableSection s = b.build();
OutputStream sec = beginSection(raw);
s.writeDelimitedTo(sec);
for (Map.Entry<Integer, String> e : u.entrySet()) {
StringTableSection.Entry.Builder x =
StringTableSection.Entry.newBuilder()
.setId(e.getKey())
.setStr(e.getValue());
x.build().writeDelimitedTo(sec);
}
endSection(sec, SectionName.STRING_TABLE);
}
@Override
public synchronized String toString() {
StringBuilder sb = new StringBuilder();
sb.append("{ codec=\"").append(compress.getImageCodec());
sb.append("\", startBlock=").append(startBlock);
sb.append(", curBlock=").append(curBlock);
sb.append(", startInode=").append(startInode);
sb.append(", curInode=").append(curInode);
sb.append(", ugi=").append(ugis);
sb.append(", blockIds=").append(blockIds);
sb.append(", offset=").append(raw.pos);
sb.append(" }");
return sb.toString();
}
static class TrackedOutputStream<T extends OutputStream>
extends FilterOutputStream {
private long pos = 0L;
TrackedOutputStream(T out) {
super(out);
}
@SuppressWarnings("unchecked")
public T getInner() {
return (T) out;
}
@Override
public void write(int b) throws IOException {
out.write(b);
++pos;
}
@Override
public void write(byte[] b) throws IOException {
write(b, 0, b.length);
}
@Override
public void write(byte[] b, int off, int len) throws IOException {
out.write(b, off, len);
pos += len;
}
@Override
public void flush() throws IOException {
super.flush();
}
@Override
public void close() throws IOException {
super.close();
}
}
/**
* Configurable options for image generation mapping pluggable components.
*/
public static class Options implements Configurable {
public static final String START_INODE = "hdfs.image.writer.start.inode";
public static final String CACHE_ENTRY = "hdfs.image.writer.cache.entries";
public static final String UGI_CLASS = "hdfs.image.writer.ugi.class";
public static final String BLOCK_RESOLVER_CLASS =
"hdfs.image.writer.blockresolver.class";
private Path outdir;
private Configuration conf;
private OutputStream outStream;
private int maxdircache;
private long startBlock;
private long startInode;
private UGIResolver ugis;
private Class<? extends UGIResolver> ugisClass;
private BlockFormat<FileRegion> blocks;
@SuppressWarnings("rawtypes")
private Class<? extends BlockFormat> blockFormatClass;
private BlockResolver blockIds;
private Class<? extends BlockResolver> blockIdsClass;
private FSImageCompression compress =
FSImageCompression.createNoopCompression();
protected Options() {
}
@Override
public void setConf(Configuration conf) {
this.conf = conf;
//long lastTxn = conf.getLong(LAST_TXN, 0L);
String def = new File("hdfs/name").toURI().toString();
outdir = new Path(conf.get(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY, def));
startBlock = conf.getLong(FixedBlockResolver.START_BLOCK, (1L << 30) + 1);
startInode = conf.getLong(START_INODE, (1L << 14) + 1);
maxdircache = conf.getInt(CACHE_ENTRY, 100);
ugisClass = conf.getClass(UGI_CLASS,
SingleUGIResolver.class, UGIResolver.class);
blockFormatClass = conf.getClass(
DFSConfigKeys.DFS_PROVIDER_BLK_FORMAT_CLASS,
NullBlockFormat.class, BlockFormat.class);
blockIdsClass = conf.getClass(BLOCK_RESOLVER_CLASS,
FixedBlockResolver.class, BlockResolver.class);
}
@Override
public Configuration getConf() {
return conf;
}
public Options output(String out) {
this.outdir = new Path(out);
return this;
}
public Options outStream(OutputStream outStream) {
this.outStream = outStream;
return this;
}
public Options codec(String codec) throws IOException {
this.compress = FSImageCompression.createCompression(getConf(), codec);
return this;
}
public Options cache(int nDirEntries) {
this.maxdircache = nDirEntries;
return this;
}
public Options ugi(UGIResolver ugis) {
this.ugis = ugis;
return this;
}
public Options ugi(Class<? extends UGIResolver> ugisClass) {
this.ugisClass = ugisClass;
return this;
}
public Options blockIds(BlockResolver blockIds) {
this.blockIds = blockIds;
return this;
}
public Options blockIds(Class<? extends BlockResolver> blockIdsClass) {
this.blockIdsClass = blockIdsClass;
return this;
}
public Options blocks(BlockFormat<FileRegion> blocks) {
this.blocks = blocks;
return this;
}
@SuppressWarnings("rawtypes")
public Options blocks(Class<? extends BlockFormat> blocksClass) {
this.blockFormatClass = blocksClass;
return this;
}
}
}

87
hadoop-tools/hadoop-fs2img/src/main/java/org/apache/hadoop/hdfs/server/namenode/NullBlockFormat.java Normal file
View File

@ -0,0 +1,87 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdfs.server.namenode;
import java.io.IOException;
import java.util.Iterator;
import java.util.NoSuchElementException;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.server.common.BlockFormat;
import org.apache.hadoop.hdfs.server.common.BlockFormat.Reader.Options;
import org.apache.hadoop.hdfs.server.common.FileRegion;
/**
* Null sink for region information emitted from FSImage.
*/
public class NullBlockFormat extends BlockFormat<FileRegion> {
@Override
public Reader<FileRegion> getReader(Options opts) throws IOException {
return new Reader<FileRegion>() {
@Override
public Iterator<FileRegion> iterator() {
return new Iterator<FileRegion>() {
@Override
public boolean hasNext() {
return false;
}
@Override
public FileRegion next() {
throw new NoSuchElementException();
}
@Override
public void remove() {
throw new UnsupportedOperationException();
}
};
}
@Override
public void close() throws IOException {
// do nothing
}
@Override
public FileRegion resolve(Block ident) throws IOException {
throw new UnsupportedOperationException();
}
};
}
@Override
public Writer<FileRegion> getWriter(Writer.Options opts) throws IOException {
return new Writer<FileRegion>() {
@Override
public void store(FileRegion token) throws IOException {
// do nothing
}
@Override
public void close() throws IOException {
// do nothing
}
};
}
@Override
public void refresh() throws IOException {
// do nothing
}
}

90
hadoop-tools/hadoop-fs2img/src/main/java/org/apache/hadoop/hdfs/server/namenode/SingleUGIResolver.java Normal file
View File

@ -0,0 +1,90 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdfs.server.namenode;
import java.io.IOException;
import org.apache.hadoop.conf.Configurable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.security.UserGroupInformation;
/**
* Map all owners/groups in external system to a single user in FSImage.
*/
public class SingleUGIResolver extends UGIResolver implements Configurable {
public static final String UID = "hdfs.image.writer.ugi.single.uid";
public static final String USER = "hdfs.image.writer.ugi.single.user";
public static final String GID = "hdfs.image.writer.ugi.single.gid";
public static final String GROUP = "hdfs.image.writer.ugi.single.group";
private int uid;
private int gid;
private String user;
private String group;
private Configuration conf;
@Override
public void setConf(Configuration conf) {
this.conf = conf;
uid = conf.getInt(UID, 0);
user = conf.get(USER);
if (null == user) {
try {
user = UserGroupInformation.getCurrentUser().getShortUserName();
} catch (IOException e) {
user = "hadoop";
}
}
gid = conf.getInt(GID, 1);
group = conf.get(GROUP);
if (null == group) {
group = user;
}
resetUGInfo();
addUser(user, uid);
addGroup(group, gid);
}
@Override
public Configuration getConf() {
return conf;
}
@Override
public String user(FileStatus s) {
return user;
}
@Override
public String group(FileStatus s) {
return group;
}
@Override
public void addUser(String name) {
//do nothing
}
@Override
public void addGroup(String name) {
//do nothing
}
}

167
hadoop-tools/hadoop-fs2img/src/main/java/org/apache/hadoop/hdfs/server/namenode/TreePath.java Normal file
View File

@ -0,0 +1,167 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdfs.server.namenode;
import java.io.IOException;
import com.google.protobuf.ByteString;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.BlockProto;
import org.apache.hadoop.hdfs.server.common.BlockFormat;
import org.apache.hadoop.hdfs.server.common.FileRegion;
import org.apache.hadoop.hdfs.server.namenode.FsImageProto.INodeSection.INode;
import org.apache.hadoop.hdfs.server.namenode.FsImageProto.INodeSection.INodeDirectory;
import org.apache.hadoop.hdfs.server.namenode.FsImageProto.INodeSection.INodeFile;
import static org.apache.hadoop.hdfs.DFSUtil.string2Bytes;
import static org.apache.hadoop.hdfs.server.namenode.DirectoryWithQuotaFeature.DEFAULT_NAMESPACE_QUOTA;
import static org.apache.hadoop.hdfs.server.namenode.DirectoryWithQuotaFeature.DEFAULT_STORAGE_SPACE_QUOTA;
/**
* Traversal cursor in external filesystem.
* TODO: generalize, move FS/FileRegion to FSTreePath
*/
public class TreePath {
private long id = -1;
private final long parentId;
private final FileStatus stat;
private final TreeWalk.TreeIterator i;
protected TreePath(FileStatus stat, long parentId, TreeWalk.TreeIterator i) {
this.i = i;
this.stat = stat;
this.parentId = parentId;
}
public FileStatus getFileStatus() {
return stat;
}
public long getParentId() {
return parentId;
}
public long getId() {
if (id < 0) {
throw new IllegalStateException();
}
return id;
}
void accept(long id) {
this.id = id;
i.onAccept(this, id);
}
public INode toINode(UGIResolver ugi, BlockResolver blk,
BlockFormat.Writer<FileRegion> out, String blockPoolID)
throws IOException {
if (stat.isFile()) {
return toFile(ugi, blk, out, blockPoolID);
} else if (stat.isDirectory()) {
return toDirectory(ugi);
} else if (stat.isSymlink()) {
throw new UnsupportedOperationException("symlinks not supported");
} else {
throw new UnsupportedOperationException("Unknown type: " + stat);
}
}
@Override
public boolean equals(Object other) {
if (!(other instanceof TreePath)) {
return false;
}
TreePath o = (TreePath) other;
return getParentId() == o.getParentId()
&& getFileStatus().equals(o.getFileStatus());
}
@Override
public int hashCode() {
long pId = getParentId() * getFileStatus().hashCode();
return (int)(pId ^ (pId >>> 32));
}
void writeBlock(long blockId, long offset, long length,
long genStamp, String blockPoolID,
BlockFormat.Writer<FileRegion> out) throws IOException {
FileStatus s = getFileStatus();
out.store(new FileRegion(blockId, s.getPath(), offset, length,
blockPoolID, genStamp));
}
INode toFile(UGIResolver ugi, BlockResolver blk,
BlockFormat.Writer<FileRegion> out, String blockPoolID)
throws IOException {
final FileStatus s = getFileStatus();
// TODO should this store resolver's user/group?
ugi.addUser(s.getOwner());
ugi.addGroup(s.getGroup());
INodeFile.Builder b = INodeFile.newBuilder()
.setReplication(blk.getReplication(s))
.setModificationTime(s.getModificationTime())
.setAccessTime(s.getAccessTime())
.setPreferredBlockSize(blk.preferredBlockSize(s))
.setPermission(ugi.resolve(s))
.setStoragePolicyID(HdfsConstants.PROVIDED_STORAGE_POLICY_ID);
//TODO: storage policy should be configurable per path; use BlockResolver
long off = 0L;
for (BlockProto block : blk.resolve(s)) {
b.addBlocks(block);
writeBlock(block.getBlockId(), off, block.getNumBytes(),
block.getGenStamp(), blockPoolID, out);
off += block.getNumBytes();
}
INode.Builder ib = INode.newBuilder()
.setType(INode.Type.FILE)
.setId(id)
.setName(ByteString.copyFrom(string2Bytes(s.getPath().getName())))
.setFile(b);
return ib.build();
}
INode toDirectory(UGIResolver ugi) {
final FileStatus s = getFileStatus();
ugi.addUser(s.getOwner());
ugi.addGroup(s.getGroup());
INodeDirectory.Builder b = INodeDirectory.newBuilder()
.setModificationTime(s.getModificationTime())
.setNsQuota(DEFAULT_NAMESPACE_QUOTA)
.setDsQuota(DEFAULT_STORAGE_SPACE_QUOTA)
.setPermission(ugi.resolve(s));
INode.Builder ib = INode.newBuilder()
.setType(INode.Type.DIRECTORY)
.setId(id)
.setName(ByteString.copyFrom(string2Bytes(s.getPath().getName())))
.setDirectory(b);
return ib.build();
}
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
sb.append("{ stat=\"").append(getFileStatus()).append("\"");
sb.append(", id=").append(getId());
sb.append(", parentId=").append(getParentId());
sb.append(", iterObjId=").append(System.identityHashCode(i));
sb.append(" }");
return sb.toString();
}
}

103
hadoop-tools/hadoop-fs2img/src/main/java/org/apache/hadoop/hdfs/server/namenode/TreeWalk.java Normal file
View File

@ -0,0 +1,103 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdfs.server.namenode;
import java.util.ArrayDeque;
import java.util.Deque;
import java.util.Iterator;
/**
* Traversal yielding a hierarchical sequence of paths.
*/
public abstract class TreeWalk implements Iterable<TreePath> {
/**
* @param path path to the node being explored.
* @param id the id of the node.
* @param iterator the {@link TreeIterator} to use.
* @return paths representing the children of the current node.
*/
protected abstract Iterable<TreePath> getChildren(
TreePath path, long id, TreeWalk.TreeIterator iterator);
public abstract TreeIterator iterator();
/**
* Enumerator class for hierarchies. Implementations SHOULD support a fork()
* operation yielding a subtree of the current cursor.
*/
public abstract class TreeIterator implements Iterator<TreePath> {
private final Deque<TreePath> pending;
TreeIterator() {
this(new ArrayDeque<TreePath>());
}
protected TreeIterator(Deque<TreePath> pending) {
this.pending = pending;
}
public abstract TreeIterator fork();
@Override
public boolean hasNext() {
return !pending.isEmpty();
}
@Override
public TreePath next() {
return pending.removeFirst();
}
@Override
public void remove() {
throw new UnsupportedOperationException();
}
protected void onAccept(TreePath p, long id) {
for (TreePath k : getChildren(p, id, this)) {
pending.addFirst(k);
}
}
/**
* @return the Deque containing the pending paths.
*/
protected Deque<TreePath> getPendingQueue() {
return pending;
}
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
sb.append("{ Treewalk=\"").append(TreeWalk.this.toString());
sb.append(", pending=[");
Iterator<TreePath> i = pending.iterator();
if (i.hasNext()) {
sb.append("\"").append(i.next()).append("\"");
}
while (i.hasNext()) {
sb.append(", \"").append(i.next()).append("\"");
}
sb.append("]");
sb.append(" }");
return sb.toString();
}
}
}

131
hadoop-tools/hadoop-fs2img/src/main/java/org/apache/hadoop/hdfs/server/namenode/UGIResolver.java Normal file
View File

@ -0,0 +1,131 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdfs.server.namenode;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.permission.FsPermission;
/**
* Pluggable class for mapping ownership and permissions from an external
* store to an FSImage.
*/
public abstract class UGIResolver {
static final int USER_STRID_OFFSET = 40;
static final int GROUP_STRID_OFFSET = 16;
static final long USER_GROUP_STRID_MASK = (1 << 24) - 1;
/**
* Permission is serialized as a 64-bit long. [0:24):[25:48):[48:64) (in Big
* Endian).
* The first and the second parts are the string ids of the user and
* group name, and the last 16 bits are the permission bits.
* @param owner name of owner
* @param group name of group
* @param permission Permission octects
* @return FSImage encoding of permissions
*/
protected final long buildPermissionStatus(
String owner, String group, short permission) {
long userId = users.get(owner);
if (0L != ((~USER_GROUP_STRID_MASK) & userId)) {
throw new IllegalArgumentException("UID must fit in 24 bits");
}
long groupId = groups.get(group);
if (0L != ((~USER_GROUP_STRID_MASK) & groupId)) {
throw new IllegalArgumentException("GID must fit in 24 bits");
}
return ((userId & USER_GROUP_STRID_MASK) << USER_STRID_OFFSET)
| ((groupId & USER_GROUP_STRID_MASK) << GROUP_STRID_OFFSET)
| permission;
}
private final Map<String, Integer> users;
private final Map<String, Integer> groups;
public UGIResolver() {
this(new HashMap<String, Integer>(), new HashMap<String, Integer>());
}
UGIResolver(Map<String, Integer> users, Map<String, Integer> groups) {
this.users = users;
this.groups = groups;
}
public Map<Integer, String> ugiMap() {
Map<Integer, String> ret = new HashMap<>();
for (Map<String, Integer> m : Arrays.asList(users, groups)) {
for (Map.Entry<String, Integer> e : m.entrySet()) {
String s = ret.put(e.getValue(), e.getKey());
if (s != null) {
throw new IllegalStateException("Duplicate mapping: " +
e.getValue() + " " + s + " " + e.getKey());
}
}
}
return ret;
}
public abstract void addUser(String name);
protected void addUser(String name, int id) {
Integer uid = users.put(name, id);
if (uid != null) {
throw new IllegalArgumentException("Duplicate mapping: " + name +
" " + uid + " " + id);
}
}
public abstract void addGroup(String name);
protected void addGroup(String name, int id) {
Integer gid = groups.put(name, id);
if (gid != null) {
throw new IllegalArgumentException("Duplicate mapping: " + name +
" " + gid + " " + id);
}
}
protected void resetUGInfo() {
users.clear();
groups.clear();
}
public long resolve(FileStatus s) {
return buildPermissionStatus(user(s), group(s), permission(s).toShort());
}
public String user(FileStatus s) {
return s.getOwner();
}
public String group(FileStatus s) {
return s.getGroup();
}
public FsPermission permission(FileStatus s) {
return s.getPermission();
}
}

23
hadoop-tools/hadoop-fs2img/src/main/java/org/apache/hadoop/hdfs/server/namenode/package-info.java Normal file
View File

@ -0,0 +1,23 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
@InterfaceAudience.Public
@InterfaceStability.Unstable
package org.apache.hadoop.hdfs.server.namenode;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;

186
hadoop-tools/hadoop-fs2img/src/test/java/org/apache/hadoop/hdfs/server/namenode/RandomTreeWalk.java Normal file
View File

@ -0,0 +1,186 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdfs.server.namenode;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import java.util.Random;
import org.apache.hadoop.fs.BlockLocation;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Path;
/**
* Random, repeatable hierarchy generator.
*/
public class RandomTreeWalk extends TreeWalk {
private final Path root;
private final long seed;
private final float depth;
private final int children;
private final Map<Long, Long> mSeed;
//private final AtomicLong blockIds = new AtomicLong(1L << 30);
RandomTreeWalk(long seed) {
this(seed, 10);
}
RandomTreeWalk(long seed, int children) {
this(seed, children, 0.15f);
}
RandomTreeWalk(long seed, int children, float depth) {
this(randomRoot(seed), seed, children, 0.15f);
}
RandomTreeWalk(Path root, long seed, int children, float depth) {
this.seed = seed;
this.depth = depth;
this.children = children;
mSeed = Collections.synchronizedMap(new HashMap<Long, Long>());
mSeed.put(-1L, seed);
this.root = root;
}
static Path randomRoot(long seed) {
Random r = new Random(seed);
String scheme;
do {
scheme = genName(r, 3, 5).toLowerCase();
} while (Character.isDigit(scheme.charAt(0)));
String authority = genName(r, 3, 15).toLowerCase();
int port = r.nextInt(1 << 13) + 1000;
return new Path(scheme, authority + ":" + port, "/");
}
@Override
public TreeIterator iterator() {
return new RandomTreeIterator(seed);
}
@Override
protected Iterable<TreePath> getChildren(TreePath p, long id,
TreeIterator walk) {
final FileStatus pFs = p.getFileStatus();
if (pFs.isFile()) {
return Collections.emptyList();
}
// seed is f(parent seed, attrib)
long cseed = mSeed.get(p.getParentId()) * p.getFileStatus().hashCode();
mSeed.put(p.getId(), cseed);
Random r = new Random(cseed);
int nChildren = r.nextInt(children);
ArrayList<TreePath> ret = new ArrayList<TreePath>();
for (int i = 0; i < nChildren; ++i) {
ret.add(new TreePath(genFileStatus(p, r), p.getId(), walk));
}
return ret;
}
FileStatus genFileStatus(TreePath parent, Random r) {
final int blocksize = 128 * (1 << 20);
final Path name;
final boolean isDir;
if (null == parent) {
name = root;
isDir = true;
} else {
Path p = parent.getFileStatus().getPath();
name = new Path(p, genName(r, 3, 10));
isDir = r.nextFloat() < depth;
}
final long len = isDir ? 0 : r.nextInt(Integer.MAX_VALUE);
final int nblocks = 0 == len ? 0 : (((int)((len - 1) / blocksize)) + 1);
BlockLocation[] blocks = genBlocks(r, nblocks, blocksize, len);
try {
return new LocatedFileStatus(new FileStatus(
len, /* long length, */
isDir, /* boolean isdir, */
1, /* int block_replication, */
blocksize, /* long blocksize, */
0L, /* long modification_time, */
0L, /* long access_time, */
null, /* FsPermission permission, */
"hadoop", /* String owner, */
"hadoop", /* String group, */
name), /* Path path */
blocks);
} catch (IOException e) {
throw new RuntimeException(e);
}
}
BlockLocation[] genBlocks(Random r, int nblocks, int blocksize, long len) {
BlockLocation[] blocks = new BlockLocation[nblocks];
if (0 == nblocks) {
return blocks;
}
for (int i = 0; i < nblocks - 1; ++i) {
blocks[i] = new BlockLocation(null, null, i * blocksize, blocksize);
}
blocks[nblocks - 1] = new BlockLocation(null, null,
(nblocks - 1) * blocksize,
0 == (len % blocksize) ? blocksize : len % blocksize);
return blocks;
}
static String genName(Random r, int min, int max) {
int len = r.nextInt(max - min + 1) + min;
char[] ret = new char[len];
while (len > 0) {
int c = r.nextInt() & 0x7F; // restrict to ASCII
if (Character.isLetterOrDigit(c)) {
ret[--len] = (char) c;
}
}
return new String(ret);
}
class RandomTreeIterator extends TreeIterator {
RandomTreeIterator() {
}
RandomTreeIterator(long seed) {
Random r = new Random(seed);
FileStatus iroot = genFileStatus(null, r);
getPendingQueue().addFirst(new TreePath(iroot, -1, this));
}
RandomTreeIterator(TreePath p) {
getPendingQueue().addFirst(
new TreePath(p.getFileStatus(), p.getParentId(), this));
}
@Override
public TreeIterator fork() {
if (getPendingQueue().isEmpty()) {
return new RandomTreeIterator();
}
return new RandomTreeIterator(getPendingQueue().removeFirst());
}
}
}

121
hadoop-tools/hadoop-fs2img/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFixedBlockResolver.java Normal file
View File

@ -0,0 +1,121 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdfs.server.namenode;
import java.util.Iterator;
import java.util.Random;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.BlockProto;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TestName;
import static org.junit.Assert.*;
/**
* Validate fixed-size block partitioning.
*/
public class TestFixedBlockResolver {
@Rule public TestName name = new TestName();
private final FixedBlockResolver blockId = new FixedBlockResolver();
@Before
public void setup() {
Configuration conf = new Configuration(false);
conf.setLong(FixedBlockResolver.BLOCKSIZE, 512L * (1L << 20));
conf.setLong(FixedBlockResolver.START_BLOCK, 512L * (1L << 20));
blockId.setConf(conf);
System.out.println(name.getMethodName());
}
@Test
public void testExactBlock() throws Exception {
FileStatus f = file(512, 256);
int nblocks = 0;
for (BlockProto b : blockId.resolve(f)) {
++nblocks;
assertEquals(512L * (1L << 20), b.getNumBytes());
}
assertEquals(1, nblocks);
FileStatus g = file(1024, 256);
nblocks = 0;
for (BlockProto b : blockId.resolve(g)) {
++nblocks;
assertEquals(512L * (1L << 20), b.getNumBytes());
}
assertEquals(2, nblocks);
FileStatus h = file(5120, 256);
nblocks = 0;
for (BlockProto b : blockId.resolve(h)) {
++nblocks;
assertEquals(512L * (1L << 20), b.getNumBytes());
}
assertEquals(10, nblocks);
}
@Test
public void testEmpty() throws Exception {
FileStatus f = file(0, 100);
Iterator<BlockProto> b = blockId.resolve(f).iterator();
assertTrue(b.hasNext());
assertEquals(0, b.next().getNumBytes());
assertFalse(b.hasNext());
}
@Test
public void testRandomFile() throws Exception {
Random r = new Random();
long seed = r.nextLong();
System.out.println("seed: " + seed);
r.setSeed(seed);
int len = r.nextInt(4096) + 512;
int blk = r.nextInt(len - 128) + 128;
FileStatus s = file(len, blk);
long nbytes = 0;
for (BlockProto b : blockId.resolve(s)) {
nbytes += b.getNumBytes();
assertTrue(512L * (1L << 20) >= b.getNumBytes());
}
assertEquals(s.getLen(), nbytes);
}
FileStatus file(long lenMB, long blocksizeMB) {
Path p = new Path("foo://bar:4344/baz/dingo");
return new FileStatus(
lenMB * (1 << 20), /* long length, */
false, /* boolean isdir, */
1, /* int block_replication, */
blocksizeMB * (1 << 20), /* long blocksize, */
0L, /* long modification_time, */
0L, /* long access_time, */
null, /* FsPermission permission, */
"hadoop", /* String owner, */
"hadoop", /* String group, */
p); /* Path path */
}
}

130
hadoop-tools/hadoop-fs2img/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestRandomTreeWalk.java Normal file
View File

@ -0,0 +1,130 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdfs.server.namenode;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
import java.util.Random;
import java.util.Set;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TestName;
import static org.junit.Assert.*;
/**
* Validate randomly generated hierarchies, including fork() support in
* base class.
*/
public class TestRandomTreeWalk {
@Rule public TestName name = new TestName();
private Random r = new Random();
@Before
public void setSeed() {
long seed = r.nextLong();
r.setSeed(seed);
System.out.println(name.getMethodName() + " seed: " + seed);
}
@Test
public void testRandomTreeWalkRepeat() throws Exception {
Set<TreePath> ns = new HashSet<>();
final long seed = r.nextLong();
RandomTreeWalk t1 = new RandomTreeWalk(seed, 10, .1f);
int i = 0;
for (TreePath p : t1) {
p.accept(i++);
assertTrue(ns.add(p));
}
RandomTreeWalk t2 = new RandomTreeWalk(seed, 10, .1f);
int j = 0;
for (TreePath p : t2) {
p.accept(j++);
assertTrue(ns.remove(p));
}
assertTrue(ns.isEmpty());
}
@Test
public void testRandomTreeWalkFork() throws Exception {
Set<FileStatus> ns = new HashSet<>();
final long seed = r.nextLong();
RandomTreeWalk t1 = new RandomTreeWalk(seed, 10, .15f);
int i = 0;
for (TreePath p : t1) {
p.accept(i++);
assertTrue(ns.add(p.getFileStatus()));
}
RandomTreeWalk t2 = new RandomTreeWalk(seed, 10, .15f);
int j = 0;
ArrayList<TreeWalk.TreeIterator> iters = new ArrayList<>();
iters.add(t2.iterator());
while (!iters.isEmpty()) {
for (TreeWalk.TreeIterator sub = iters.remove(iters.size() - 1);
sub.hasNext();) {
TreePath p = sub.next();
if (0 == (r.nextInt() % 4)) {
iters.add(sub.fork());
Collections.shuffle(iters, r);
}
p.accept(j++);
assertTrue(ns.remove(p.getFileStatus()));
}
}
assertTrue(ns.isEmpty());
}
@Test
public void testRandomRootWalk() throws Exception {
Set<FileStatus> ns = new HashSet<>();
final long seed = r.nextLong();
Path root = new Path("foo://bar:4344/dingos");
String sroot = root.toString();
int nroot = sroot.length();
RandomTreeWalk t1 = new RandomTreeWalk(root, seed, 10, .1f);
int i = 0;
for (TreePath p : t1) {
p.accept(i++);
FileStatus stat = p.getFileStatus();
assertTrue(ns.add(stat));
assertEquals(sroot, stat.getPath().toString().substring(0, nroot));
}
RandomTreeWalk t2 = new RandomTreeWalk(root, seed, 10, .1f);
int j = 0;
for (TreePath p : t2) {
p.accept(j++);
FileStatus stat = p.getFileStatus();
assertTrue(ns.remove(stat));
assertEquals(sroot, stat.getPath().toString().substring(0, nroot));
}
assertTrue(ns.isEmpty());
}
}

148
hadoop-tools/hadoop-fs2img/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSingleUGIResolver.java Normal file
View File

@ -0,0 +1,148 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdfs.server.namenode;
import java.io.IOException;
import java.util.Map;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.security.UserGroupInformation;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TestName;
import static org.junit.Assert.*;
/**
* Validate resolver assigning all paths to a single owner/group.
*/
public class TestSingleUGIResolver {
@Rule public TestName name = new TestName();
private static final int TESTUID = 10101;
private static final int TESTGID = 10102;
private static final String TESTUSER = "tenaqvyybdhragqvatbf";
private static final String TESTGROUP = "tnyybcvatlnxf";
private SingleUGIResolver ugi = new SingleUGIResolver();
@Before
public void setup() {
Configuration conf = new Configuration(false);
conf.setInt(SingleUGIResolver.UID, TESTUID);
conf.setInt(SingleUGIResolver.GID, TESTGID);
conf.set(SingleUGIResolver.USER, TESTUSER);
conf.set(SingleUGIResolver.GROUP, TESTGROUP);
ugi.setConf(conf);
System.out.println(name.getMethodName());
}
@Test
public void testRewrite() {
FsPermission p1 = new FsPermission((short)0755);
match(ugi.resolve(file("dingo", "dingo", p1)), p1);
match(ugi.resolve(file(TESTUSER, "dingo", p1)), p1);
match(ugi.resolve(file("dingo", TESTGROUP, p1)), p1);
match(ugi.resolve(file(TESTUSER, TESTGROUP, p1)), p1);
FsPermission p2 = new FsPermission((short)0x8000);
match(ugi.resolve(file("dingo", "dingo", p2)), p2);
match(ugi.resolve(file(TESTUSER, "dingo", p2)), p2);
match(ugi.resolve(file("dingo", TESTGROUP, p2)), p2);
match(ugi.resolve(file(TESTUSER, TESTGROUP, p2)), p2);
Map<Integer, String> ids = ugi.ugiMap();
assertEquals(2, ids.size());
assertEquals(TESTUSER, ids.get(10101));
assertEquals(TESTGROUP, ids.get(10102));
}
@Test
public void testDefault() {
String user;
try {
user = UserGroupInformation.getCurrentUser().getShortUserName();
} catch (IOException e) {
user = "hadoop";
}
Configuration conf = new Configuration(false);
ugi.setConf(conf);
Map<Integer, String> ids = ugi.ugiMap();
assertEquals(2, ids.size());
assertEquals(user, ids.get(0));
assertEquals(user, ids.get(1));
}
@Test(expected=IllegalArgumentException.class)
public void testInvalidUid() {
Configuration conf = ugi.getConf();
conf.setInt(SingleUGIResolver.UID, (1 << 24) + 1);
ugi.setConf(conf);
ugi.resolve(file(TESTUSER, TESTGROUP, new FsPermission((short)0777)));
}
@Test(expected=IllegalArgumentException.class)
public void testInvalidGid() {
Configuration conf = ugi.getConf();
conf.setInt(SingleUGIResolver.GID, (1 << 24) + 1);
ugi.setConf(conf);
ugi.resolve(file(TESTUSER, TESTGROUP, new FsPermission((short)0777)));
}
@Test(expected=IllegalStateException.class)
public void testDuplicateIds() {
Configuration conf = new Configuration(false);
conf.setInt(SingleUGIResolver.UID, 4344);
conf.setInt(SingleUGIResolver.GID, 4344);
conf.set(SingleUGIResolver.USER, TESTUSER);
conf.set(SingleUGIResolver.GROUP, TESTGROUP);
ugi.setConf(conf);
ugi.ugiMap();
}
static void match(long encoded, FsPermission p) {
assertEquals(p, new FsPermission((short)(encoded & 0xFFFF)));
long uid = (encoded >>> UGIResolver.USER_STRID_OFFSET);
uid &= UGIResolver.USER_GROUP_STRID_MASK;
assertEquals(TESTUID, uid);
long gid = (encoded >>> UGIResolver.GROUP_STRID_OFFSET);
gid &= UGIResolver.USER_GROUP_STRID_MASK;
assertEquals(TESTGID, gid);
}
static FileStatus file(String user, String group, FsPermission perm) {
Path p = new Path("foo://bar:4344/baz/dingo");
return new FileStatus(
4344 * (1 << 20), /* long length, */
false, /* boolean isdir, */
1, /* int block_replication, */
256 * (1 << 20), /* long blocksize, */
0L, /* long modification_time, */
0L, /* long access_time, */
perm, /* FsPermission permission, */
user, /* String owner, */
group, /* String group, */
p); /* Path path */
}
}

24
hadoop-tools/hadoop-fs2img/src/test/resources/log4j.properties Normal file
View File

@ -0,0 +1,24 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# log4j configuration used during build and unit tests
log4j.rootLogger=INFO,stdout
log4j.threshold=ALL
log4j.appender.stdout=org.apache.log4j.ConsoleAppender
log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %-5p [%t]: %c{2} (%F:%M(%L)) - %m%n

6
hadoop-tools/hadoop-tools-dist/pom.xml
View File

@ -128,6 +128,12 @@
<scope>compile</scope>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-fs2img</artifactId>
<scope>compile</scope>
<version>${project.version}</version>
</dependency>
</dependencies>
<build>

1
hadoop-tools/pom.xml
View File

@ -48,6 +48,7 @@
<module>hadoop-kafka</module>
<module>hadoop-azure-datalake</module>
<module>hadoop-aliyun</module>
<module>hadoop-fs2img</module>
</modules>
<build>