big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-16654:Delete hadoop-ozone and hadoop-hdds subprojects from apache trunk

Browse Source 
Signed-off-by: Dinesh Chitlangia <dineshchitlangia@gmail.com>
This commit is contained in:
Sandeep Nemuri 2019-10-24 20:13:44 +05:30 committed by Dinesh Chitlangia
parent b2cc8b6b4a
commit 9f0610fb83
2334 changed files with 0 additions and 349955 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
BUILDING.txt
View File

@ -104,8 +104,6 @@ Maven main modules:
- hadoop-hdfs-project (Hadoop HDFS)
- hadoop-yarn-project (Hadoop YARN)
- hadoop-mapreduce-project (Hadoop MapReduce)
- hadoop-ozone (Hadoop Ozone)
- hadoop-hdds (Hadoop Distributed Data Store)
- hadoop-tools (Hadoop tools like Streaming, Distcp, etc.)
- hadoop-dist (Hadoop distribution assembler)
- hadoop-client-modules (Hadoop client modules)

2
hadoop-assemblies/src/main/resources/assemblies/hadoop-src.xml
View File

@ -56,8 +56,6 @@
<exclude>**/build/**</exclude>
<exclude>**/file:/**</exclude>
<exclude>**/SecurityAuth.audit*</exclude>
<exclude>hadoop-ozone/**</exclude>
<exclude>hadoop-hdds/**</exclude>
<exclude>hadoop-submarine/**</exclude>
</excludes>
</fileSet>

44
hadoop-hdds/client/pom.xml
View File

@ -1,44 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. See accompanying LICENSE file.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdds</artifactId>
<version>0.5.0-SNAPSHOT</version>
</parent>
<artifactId>hadoop-hdds-client</artifactId>
<version>0.5.0-SNAPSHOT</version>
<description>Apache Hadoop Distributed Data Store Client Library</description>
<name>Apache Hadoop HDDS Client</name>
<packaging>jar</packaging>
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdds-common</artifactId>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-all</artifactId>
</dependency>
</dependencies>
</project>

65
hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/ClientCredentialInterceptor.java
View File

@ -1,65 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements.  See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership.  The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License.  You may obtain a copy of the License at
*
*      http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdds.scm;
import org.apache.ratis.thirdparty.io.grpc.CallOptions;
import org.apache.ratis.thirdparty.io.grpc.Channel;
import org.apache.ratis.thirdparty.io.grpc.ClientCall;
import org.apache.ratis.thirdparty.io.grpc.ClientInterceptor;
import org.apache.ratis.thirdparty.io.grpc.ForwardingClientCall;
import org.apache.ratis.thirdparty.io.grpc.Metadata;
import org.apache.ratis.thirdparty.io.grpc.MethodDescriptor;
import static org.apache.hadoop.ozone.OzoneConsts.OBT_METADATA_KEY;
import static org.apache.hadoop.ozone.OzoneConsts.USER_METADATA_KEY;
/**
* GRPC client interceptor for ozone block token.
*/
public class ClientCredentialInterceptor implements ClientInterceptor {
private final String user;
private final String token;
public ClientCredentialInterceptor(String user, String token) {
this.user = user;
this.token = token;
}
@Override
public <ReqT, RespT> ClientCall<ReqT, RespT> interceptCall(
MethodDescriptor<ReqT, RespT> method,
CallOptions callOptions,
Channel next) {
return new ForwardingClientCall.SimpleForwardingClientCall<ReqT, RespT>(
next.newCall(method, callOptions)) {
@Override
public void start(Listener<RespT> responseListener, Metadata headers) {
if (token != null) {
headers.put(OBT_METADATA_KEY, token);
}
if (user != null) {
headers.put(USER_METADATA_KEY, user);
}
super.start(responseListener, headers);
}
};
}
}

466
hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java
View File

@ -1,466 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdds.scm;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdds.HddsUtils;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandRequestProto;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandResponseProto;
import org.apache.hadoop.hdds.protocol.datanode.proto.XceiverClientProtocolServiceGrpc;
import org.apache.hadoop.hdds.protocol.datanode.proto.XceiverClientProtocolServiceGrpc.XceiverClientProtocolServiceStub;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.scm.client.HddsClientUtils;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
import org.apache.hadoop.hdds.scm.storage.CheckedBiFunction;
import org.apache.hadoop.hdds.security.exception.SCMSecurityException;
import org.apache.hadoop.hdds.security.x509.SecurityConfig;
import org.apache.hadoop.hdds.tracing.GrpcClientInterceptor;
import org.apache.hadoop.hdds.tracing.TracingUtil;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.Time;
import io.opentracing.Scope;
import io.opentracing.util.GlobalTracer;
import org.apache.ratis.thirdparty.io.grpc.ManagedChannel;
import org.apache.ratis.thirdparty.io.grpc.Status;
import org.apache.ratis.thirdparty.io.grpc.netty.GrpcSslContexts;
import org.apache.ratis.thirdparty.io.grpc.netty.NettyChannelBuilder;
import org.apache.ratis.thirdparty.io.grpc.stub.StreamObserver;
import org.apache.ratis.thirdparty.io.netty.handler.ssl.SslContextBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.security.cert.X509Certificate;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Semaphore;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
/**
* A Client for the storageContainer protocol for read object data.
*/
public class XceiverClientGrpc extends XceiverClientSpi {
static final Logger LOG = LoggerFactory.getLogger(XceiverClientGrpc.class);
private static final String COMPONENT = "dn";
private final Pipeline pipeline;
private final Configuration config;
private Map<UUID, XceiverClientProtocolServiceStub> asyncStubs;
private XceiverClientMetrics metrics;
private Map<UUID, ManagedChannel> channels;
private final Semaphore semaphore;
private boolean closed = false;
private SecurityConfig secConfig;
private final boolean topologyAwareRead;
private X509Certificate caCert;
/**
* Constructs a client that can communicate with the Container framework on
* data nodes.
*
* @param pipeline - Pipeline that defines the machines.
* @param config -- Ozone Config
* @param caCert - SCM ca certificate.
*/
public XceiverClientGrpc(Pipeline pipeline, Configuration config,
X509Certificate caCert) {
super();
Preconditions.checkNotNull(pipeline);
Preconditions.checkNotNull(config);
this.pipeline = pipeline;
this.config = config;
this.secConfig = new SecurityConfig(config);
this.semaphore =
new Semaphore(HddsClientUtils.getMaxOutstandingRequests(config));
this.metrics = XceiverClientManager.getXceiverClientMetrics();
this.channels = new HashMap<>();
this.asyncStubs = new HashMap<>();
this.topologyAwareRead = config.getBoolean(
OzoneConfigKeys.OZONE_NETWORK_TOPOLOGY_AWARE_READ_KEY,
OzoneConfigKeys.OZONE_NETWORK_TOPOLOGY_AWARE_READ_DEFAULT);
this.caCert = caCert;
}
/**
* Constructs a client that can communicate with the Container framework on
* data nodes.
*
* @param pipeline - Pipeline that defines the machines.
* @param config -- Ozone Config
*/
public XceiverClientGrpc(Pipeline pipeline, Configuration config) {
this(pipeline, config, null);
}
/**
* To be used when grpc token is not enabled.
*/
@Override
public void connect() throws Exception {
// connect to the closest node, if closest node doesn't exist, delegate to
// first node, which is usually the leader in the pipeline.
DatanodeDetails dn = topologyAwareRead ? this.pipeline.getClosestNode() :
this.pipeline.getFirstNode();
// just make a connection to the picked datanode at the beginning
connectToDatanode(dn, null);
}
/**
* Passed encoded token to GRPC header when security is enabled.
*/
@Override
public void connect(String encodedToken) throws Exception {
// connect to the closest node, if closest node doesn't exist, delegate to
// first node, which is usually the leader in the pipeline.
DatanodeDetails dn = topologyAwareRead ? this.pipeline.getClosestNode() :
this.pipeline.getFirstNode();
// just make a connection to the picked datanode at the beginning
connectToDatanode(dn, encodedToken);
}
private void connectToDatanode(DatanodeDetails dn, String encodedToken)
throws IOException {
// read port from the data node, on failure use default configured
// port.
int port = dn.getPort(DatanodeDetails.Port.Name.STANDALONE).getValue();
if (port == 0) {
port = config.getInt(OzoneConfigKeys.DFS_CONTAINER_IPC_PORT,
OzoneConfigKeys.DFS_CONTAINER_IPC_PORT_DEFAULT);
}
// Add credential context to the client call
String userName = UserGroupInformation.getCurrentUser().getShortUserName();
if (LOG.isDebugEnabled()) {
LOG.debug("Nodes in pipeline : {}", pipeline.getNodes().toString());
LOG.debug("Connecting to server : {}", dn.getIpAddress());
}
NettyChannelBuilder channelBuilder =
NettyChannelBuilder.forAddress(dn.getIpAddress(), port).usePlaintext()
.maxInboundMessageSize(OzoneConsts.OZONE_SCM_CHUNK_MAX_SIZE)
.intercept(new ClientCredentialInterceptor(userName, encodedToken),
new GrpcClientInterceptor());
if (secConfig.isGrpcTlsEnabled()) {
SslContextBuilder sslContextBuilder = GrpcSslContexts.forClient();
if (caCert != null) {
sslContextBuilder.trustManager(caCert);
}
if (secConfig.useTestCert()) {
channelBuilder.overrideAuthority("localhost");
}
channelBuilder.useTransportSecurity().
sslContext(sslContextBuilder.build());
} else {
channelBuilder.usePlaintext();
}
ManagedChannel channel = channelBuilder.build();
XceiverClientProtocolServiceStub asyncStub =
XceiverClientProtocolServiceGrpc.newStub(channel);
asyncStubs.put(dn.getUuid(), asyncStub);
channels.put(dn.getUuid(), channel);
}
/**
* Returns if the xceiver client connects to all servers in the pipeline.
*
* @return True if the connection is alive, false otherwise.
*/
@VisibleForTesting
public boolean isConnected(DatanodeDetails details) {
return isConnected(channels.get(details.getUuid()));
}
private boolean isConnected(ManagedChannel channel) {
return channel != null && !channel.isTerminated() && !channel.isShutdown();
}
@Override
public void close() {
closed = true;
for (ManagedChannel channel : channels.values()) {
channel.shutdownNow();
try {
channel.awaitTermination(60, TimeUnit.MINUTES);
} catch (Exception e) {
LOG.error("Unexpected exception while waiting for channel termination",
e);
}
}
}
@Override
public Pipeline getPipeline() {
return pipeline;
}
@Override
public ContainerCommandResponseProto sendCommand(
ContainerCommandRequestProto request) throws IOException {
try {
XceiverClientReply reply;
reply = sendCommandWithTraceIDAndRetry(request, null);
ContainerCommandResponseProto responseProto = reply.getResponse().get();
return responseProto;
} catch (ExecutionException | InterruptedException e) {
throw new IOException("Failed to execute command " + request, e);
}
}
@Override
public ContainerCommandResponseProto sendCommand(
ContainerCommandRequestProto request, List<CheckedBiFunction> validators)
throws IOException {
try {
XceiverClientReply reply;
reply = sendCommandWithTraceIDAndRetry(request, validators);
ContainerCommandResponseProto responseProto = reply.getResponse().get();
return responseProto;
} catch (ExecutionException | InterruptedException e) {
throw new IOException("Failed to execute command " + request, e);
}
}
private XceiverClientReply sendCommandWithTraceIDAndRetry(
ContainerCommandRequestProto request, List<CheckedBiFunction> validators)
throws IOException {
try (Scope scope = GlobalTracer.get()
.buildSpan("XceiverClientGrpc." + request.getCmdType().name())
.startActive(true)) {
ContainerCommandRequestProto finalPayload =
ContainerCommandRequestProto.newBuilder(request)
.setTraceID(TracingUtil.exportCurrentSpan()).build();
return sendCommandWithRetry(finalPayload, validators);
}
}
private XceiverClientReply sendCommandWithRetry(
ContainerCommandRequestProto request, List<CheckedBiFunction> validators)
throws IOException {
ContainerCommandResponseProto responseProto = null;
IOException ioException = null;
// In case of an exception or an error, we will try to read from the
// datanodes in the pipeline in a round robin fashion.
// TODO: cache the correct leader info in here, so that any subsequent calls
// should first go to leader
XceiverClientReply reply = new XceiverClientReply(null);
List<DatanodeDetails> datanodeList;
if ((request.getCmdType() == ContainerProtos.Type.ReadChunk ||
request.getCmdType() == ContainerProtos.Type.GetSmallFile) &&
topologyAwareRead) {
datanodeList = pipeline.getNodesInOrder();
} else {
datanodeList = pipeline.getNodes();
// Shuffle datanode list so that clients do not read in the same order
// every time.
Collections.shuffle(datanodeList);
}
for (DatanodeDetails dn : datanodeList) {
try {
if (LOG.isDebugEnabled()) {
LOG.debug("Executing command " + request + " on datanode " + dn);
}
// In case the command gets retried on a 2nd datanode,
// sendCommandAsyncCall will create a new channel and async stub
// in case these don't exist for the specific datanode.
reply.addDatanode(dn);
responseProto = sendCommandAsync(request, dn).getResponse().get();
if (validators != null && !validators.isEmpty()) {
for (CheckedBiFunction validator : validators) {
validator.apply(request, responseProto);
}
}
break;
} catch (ExecutionException | InterruptedException | IOException e) {
LOG.error("Failed to execute command " + request + " on datanode " + dn
.getUuidString(), e);
if (!(e instanceof IOException)) {
if (Status.fromThrowable(e.getCause()).getCode()
== Status.UNAUTHENTICATED.getCode()) {
throw new SCMSecurityException("Failed to authenticate with "
+ "GRPC XceiverServer with Ozone block token.");
}
ioException = new IOException(e);
} else {
ioException = (IOException) e;
}
responseProto = null;
}
}
if (responseProto != null) {
reply.setResponse(CompletableFuture.completedFuture(responseProto));
return reply;
} else {
Preconditions.checkNotNull(ioException);
LOG.error("Failed to execute command {} on the pipeline {}.", request,
pipeline);
throw ioException;
}
}
// TODO: for a true async API, once the waitable future while executing
// the command on one channel fails, it should be retried asynchronously
// on the future Task for all the remaining datanodes.
// Note: this Async api is not used currently used in any active I/O path.
// In case it gets used, the asynchronous retry logic needs to be plugged
// in here.
/**
* Sends a given command to server gets a waitable future back.
*
* @param request Request
* @return Response to the command
* @throws IOException
*/
@Override
public XceiverClientReply sendCommandAsync(
ContainerCommandRequestProto request)
throws IOException, ExecutionException, InterruptedException {
try (Scope scope = GlobalTracer.get()
.buildSpan("XceiverClientGrpc." + request.getCmdType().name())
.startActive(true)) {
ContainerCommandRequestProto finalPayload =
ContainerCommandRequestProto.newBuilder(request)
.setTraceID(TracingUtil.exportCurrentSpan())
.build();
XceiverClientReply asyncReply =
sendCommandAsync(finalPayload, pipeline.getFirstNode());
// TODO : for now make this API sync in nature as async requests are
// served out of order over XceiverClientGrpc. This needs to be fixed
// if this API is to be used for I/O path. Currently, this is not
// used for Read/Write Operation but for tests.
if (!HddsUtils.isReadOnly(request)) {
asyncReply.getResponse().get();
}
return asyncReply;
}
}
private XceiverClientReply sendCommandAsync(
ContainerCommandRequestProto request, DatanodeDetails dn)
throws IOException, ExecutionException, InterruptedException {
if (closed) {
throw new IOException("This channel is not connected.");
}
UUID dnId = dn.getUuid();
ManagedChannel channel = channels.get(dnId);
// If the channel doesn't exist for this specific datanode or the channel
// is closed, just reconnect
String token = request.getEncodedToken();
if (!isConnected(channel)) {
reconnect(dn, token);
}
if (LOG.isDebugEnabled()) {
LOG.debug("Send command {} to datanode {}",
request.getCmdType().toString(), dn.getNetworkFullPath());
}
final CompletableFuture<ContainerCommandResponseProto> replyFuture =
new CompletableFuture<>();
semaphore.acquire();
long requestTime = Time.monotonicNowNanos();
metrics.incrPendingContainerOpsMetrics(request.getCmdType());
// create a new grpc stream for each non-async call.
// TODO: for async calls, we should reuse StreamObserver resources.
final StreamObserver<ContainerCommandRequestProto> requestObserver =
asyncStubs.get(dnId)
.send(new StreamObserver<ContainerCommandResponseProto>() {
@Override
public void onNext(ContainerCommandResponseProto value) {
replyFuture.complete(value);
metrics.decrPendingContainerOpsMetrics(request.getCmdType());
metrics.addContainerOpsLatency(request.getCmdType(),
Time.monotonicNowNanos() - requestTime);
semaphore.release();
}
@Override
public void onError(Throwable t) {
replyFuture.completeExceptionally(t);
metrics.decrPendingContainerOpsMetrics(request.getCmdType());
metrics.addContainerOpsLatency(request.getCmdType(),
Time.monotonicNowNanos() - requestTime);
semaphore.release();
}
@Override
public void onCompleted() {
if (!replyFuture.isDone()) {
replyFuture.completeExceptionally(new IOException(
"Stream completed but no reply for request " + request));
}
}
});
requestObserver.onNext(request);
requestObserver.onCompleted();
return new XceiverClientReply(replyFuture);
}
private void reconnect(DatanodeDetails dn, String encodedToken)
throws IOException {
ManagedChannel channel;
try {
connectToDatanode(dn, encodedToken);
channel = channels.get(dn.getUuid());
} catch (Exception e) {
LOG.error("Error while connecting: ", e);
throw new IOException(e);
}
if (channel == null || !isConnected(channel)) {
throw new IOException("This channel is not connected.");
}
}
@Override
public XceiverClientReply watchForCommit(long index, long timeout)
throws InterruptedException, ExecutionException, TimeoutException,
IOException {
// there is no notion of watch for commit index in standalone pipeline
return null;
};
public long getReplicatedMinCommitIndex() {
return 0;
}
/**
* Returns pipeline Type.
*
* @return - Stand Alone as the type.
*/
@Override
public HddsProtos.ReplicationType getPipelineType() {
return HddsProtos.ReplicationType.STAND_ALONE;
}
}

390
hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientManager.java
View File

@ -1,390 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdds.scm;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.RemovalListener;
import com.google.common.cache.RemovalNotification;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdds.conf.Config;
import org.apache.hadoop.hdds.conf.ConfigGroup;
import org.apache.hadoop.hdds.conf.ConfigType;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.security.exception.SCMSecurityException;
import org.apache.hadoop.hdds.security.x509.certificate.utils.CertificateCodec;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.OzoneSecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.ratis.thirdparty.com.google.protobuf.ByteString;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.Closeable;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.security.cert.CertificateException;
import java.security.cert.X509Certificate;
import java.util.concurrent.Callable;
import java.util.concurrent.TimeUnit;
import java.util.function.Function;
import static java.util.concurrent.TimeUnit.MILLISECONDS;
import static org.apache.hadoop.hdds.conf.ConfigTag.OZONE;
import static org.apache.hadoop.hdds.conf.ConfigTag.PERFORMANCE;
/**
* XceiverClientManager is responsible for the lifecycle of XceiverClient
* instances. Callers use this class to acquire an XceiverClient instance
* connected to the desired container pipeline. When done, the caller also uses
* this class to release the previously acquired XceiverClient instance.
*
*
* This class caches connection to container for reuse purpose, such that
* accessing same container frequently will be through the same connection
* without reestablishing connection. But the connection will be closed if
* not being used for a period of time.
*/
public class XceiverClientManager implements Closeable {
private static final Logger LOG =
LoggerFactory.getLogger(XceiverClientManager.class);
//TODO : change this to SCM configuration class
private final Configuration conf;
private final Cache<String, XceiverClientSpi> clientCache;
private final boolean useRatis;
private X509Certificate caCert;
private static XceiverClientMetrics metrics;
private boolean isSecurityEnabled;
private final boolean topologyAwareRead;
/**
* Creates a new XceiverClientManager for non secured ozone cluster.
* For security enabled ozone cluster, client should use the other constructor
* with a valid ca certificate in pem string format.
*
* @param conf configuration
*/
public XceiverClientManager(Configuration conf) throws IOException {
this(conf, OzoneConfiguration.of(conf).getObject(ScmClientConfig.class),
null);
}
public XceiverClientManager(Configuration conf, ScmClientConfig clientConf,
String caCertPem) throws IOException {
Preconditions.checkNotNull(clientConf);
Preconditions.checkNotNull(conf);
long staleThresholdMs = clientConf.getStaleThreshold(MILLISECONDS);
this.useRatis = conf.getBoolean(
ScmConfigKeys.DFS_CONTAINER_RATIS_ENABLED_KEY,
ScmConfigKeys.DFS_CONTAINER_RATIS_ENABLED_DEFAULT);
this.conf = conf;
this.isSecurityEnabled = OzoneSecurityUtil.isSecurityEnabled(conf);
if (isSecurityEnabled) {
Preconditions.checkNotNull(caCertPem);
try {
this.caCert = CertificateCodec.getX509Cert(caCertPem);
} catch (CertificateException ex) {
throw new SCMSecurityException("Error: Fail to get SCM CA certificate",
ex);
}
}
this.clientCache = CacheBuilder.newBuilder()
.expireAfterAccess(staleThresholdMs, MILLISECONDS)
.maximumSize(clientConf.getMaxSize())
.removalListener(
new RemovalListener<String, XceiverClientSpi>() {
@Override
public void onRemoval(
RemovalNotification<String, XceiverClientSpi>
removalNotification) {
synchronized (clientCache) {
// Mark the entry as evicted
XceiverClientSpi info = removalNotification.getValue();
info.setEvicted();
}
}
}).build();
topologyAwareRead = conf.getBoolean(
OzoneConfigKeys.OZONE_NETWORK_TOPOLOGY_AWARE_READ_KEY,
OzoneConfigKeys.OZONE_NETWORK_TOPOLOGY_AWARE_READ_DEFAULT);
}
@VisibleForTesting
public Cache<String, XceiverClientSpi> getClientCache() {
return clientCache;
}
/**
* Acquires a XceiverClientSpi connected to a container capable of
* storing the specified key.
*
* If there is already a cached XceiverClientSpi, simply return
* the cached otherwise create a new one.
*
* @param pipeline the container pipeline for the client connection
* @return XceiverClientSpi connected to a container
* @throws IOException if a XceiverClientSpi cannot be acquired
*/
public XceiverClientSpi acquireClient(Pipeline pipeline)
throws IOException {
return acquireClient(pipeline, false);
}
/**
* Acquires a XceiverClientSpi connected to a container for read.
*
* If there is already a cached XceiverClientSpi, simply return
* the cached otherwise create a new one.
*
* @param pipeline the container pipeline for the client connection
* @return XceiverClientSpi connected to a container
* @throws IOException if a XceiverClientSpi cannot be acquired
*/
public XceiverClientSpi acquireClientForReadData(Pipeline pipeline)
throws IOException {
return acquireClient(pipeline, true);
}
private XceiverClientSpi acquireClient(Pipeline pipeline, boolean read)
throws IOException {
Preconditions.checkNotNull(pipeline);
Preconditions.checkArgument(pipeline.getNodes() != null);
Preconditions.checkArgument(!pipeline.getNodes().isEmpty());
synchronized (clientCache) {
XceiverClientSpi info = getClient(pipeline, read);
info.incrementReference();
return info;
}
}
/**
* Releases a XceiverClientSpi after use.
*
* @param client client to release
* @param invalidateClient if true, invalidates the client in cache
*/
public void releaseClient(XceiverClientSpi client, boolean invalidateClient) {
releaseClient(client, invalidateClient, false);
}
/**
* Releases a read XceiverClientSpi after use.
*
* @param client client to release
* @param invalidateClient if true, invalidates the client in cache
*/
public void releaseClientForReadData(XceiverClientSpi client,
boolean invalidateClient) {
releaseClient(client, invalidateClient, true);
}
private void releaseClient(XceiverClientSpi client, boolean invalidateClient,
boolean read) {
Preconditions.checkNotNull(client);
synchronized (clientCache) {
client.decrementReference();
if (invalidateClient) {
Pipeline pipeline = client.getPipeline();
String key = getPipelineCacheKey(pipeline, read);
XceiverClientSpi cachedClient = clientCache.getIfPresent(key);
if (cachedClient == client) {
clientCache.invalidate(key);
}
}
}
}
private XceiverClientSpi getClient(Pipeline pipeline, boolean forRead)
throws IOException {
HddsProtos.ReplicationType type = pipeline.getType();
try {
// create different client for read different pipeline node based on
// network topology
String key = getPipelineCacheKey(pipeline, forRead);
// Append user short name to key to prevent a different user
// from using same instance of xceiverClient.
key = isSecurityEnabled ?
key + UserGroupInformation.getCurrentUser().getShortUserName() : key;
return clientCache.get(key, new Callable<XceiverClientSpi>() {
@Override
public XceiverClientSpi call() throws Exception {
XceiverClientSpi client = null;
switch (type) {
case RATIS:
client = XceiverClientRatis.newXceiverClientRatis(pipeline, conf,
caCert);
client.connect();
break;
case STAND_ALONE:
client = new XceiverClientGrpc(pipeline, conf, caCert);
break;
case CHAINED:
default:
throw new IOException("not implemented" + pipeline.getType());
}
return client;
}
});
} catch (Exception e) {
throw new IOException(
"Exception getting XceiverClient: " + e.toString(), e);
}
}
private String getPipelineCacheKey(Pipeline pipeline, boolean forRead) {
String key = pipeline.getId().getId().toString() + pipeline.getType();
if (topologyAwareRead && forRead) {
try {
key += pipeline.getClosestNode().getHostName();
} catch (IOException e) {
LOG.error("Failed to get closest node to create pipeline cache key:" +
e.getMessage());
}
}
return key;
}
/**
* Close and remove all the cached clients.
*/
@Override
public void close() {
//closing is done through RemovalListener
clientCache.invalidateAll();
clientCache.cleanUp();
if (metrics != null) {
metrics.unRegister();
}
}
/**
* Tells us if Ratis is enabled for this cluster.
* @return True if Ratis is enabled.
*/
public boolean isUseRatis() {
return useRatis;
}
/**
* Returns hard coded 3 as replication factor.
* @return 3
*/
public HddsProtos.ReplicationFactor getFactor() {
if(isUseRatis()) {
return HddsProtos.ReplicationFactor.THREE;
}
return HddsProtos.ReplicationFactor.ONE;
}
/**
* Returns the default replication type.
* @return Ratis or Standalone
*/
public HddsProtos.ReplicationType getType() {
// TODO : Fix me and make Ratis default before release.
// TODO: Remove this as replication factor and type are pipeline properties
if(isUseRatis()) {
return HddsProtos.ReplicationType.RATIS;
}
return HddsProtos.ReplicationType.STAND_ALONE;
}
public Function<ByteBuffer, ByteString> byteBufferToByteStringConversion(){
return ByteStringConversion.createByteBufferConversion(conf);
}
/**
* Get xceiver client metric.
*/
public synchronized static XceiverClientMetrics getXceiverClientMetrics() {
if (metrics == null) {
metrics = XceiverClientMetrics.create();
}
return metrics;
}
/**
* Configuration for HDDS client.
*/
@ConfigGroup(prefix = "scm.container.client")
public static class ScmClientConfig {
private int maxSize;
private long staleThreshold;
private int maxOutstandingRequests;
public long getStaleThreshold(TimeUnit unit) {
return unit.convert(staleThreshold, MILLISECONDS);
}
@Config(key = "idle.threshold",
type = ConfigType.TIME, timeUnit = MILLISECONDS,
defaultValue = "10s",
tags = { OZONE, PERFORMANCE },
description =
"In the standalone pipelines, the SCM clients use netty to "
+ " communicate with the container. It also uses connection pooling"
+ " to reduce client side overheads. This allows a connection to"
+ " stay idle for a while before the connection is closed."
)
public void setStaleThreshold(long staleThreshold) {
this.staleThreshold = staleThreshold;
}
public int getMaxSize() {
return maxSize;
}
@Config(key = "max.size",
defaultValue = "256",
tags = { OZONE, PERFORMANCE },
description =
"Controls the maximum number of connections that are cached via"
+ " client connection pooling. If the number of connections"
+ " exceed this count, then the oldest idle connection is evicted."
)
public void setMaxSize(int maxSize) {
this.maxSize = maxSize;
}
public int getMaxOutstandingRequests() {
return maxOutstandingRequests;
}
@Config(key = "max.outstanding.requests",
defaultValue = "100",
tags = { OZONE, PERFORMANCE },
description =
"Controls the maximum number of outstanding async requests that can"
+ " be handled by the Standalone as well as Ratis client."
)
public void setMaxOutstandingRequests(int maxOutstandingRequests) {
this.maxOutstandingRequests = maxOutstandingRequests;
}
}
}

113
hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientMetrics.java
View File

@ -1,113 +0,0 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdds.scm;
import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
import org.apache.hadoop.metrics2.MetricsSystem;
import org.apache.hadoop.metrics2.annotation.Metric;
import org.apache.hadoop.metrics2.annotation.Metrics;
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
import org.apache.hadoop.metrics2.lib.MetricsRegistry;
import org.apache.hadoop.metrics2.lib.MutableCounterLong;
import org.apache.hadoop.metrics2.lib.MutableRate;
/**
* The client metrics for the Storage Container protocol.
*/
@InterfaceAudience.Private
@Metrics(about = "Storage Container Client Metrics", context = "dfs")
public class XceiverClientMetrics {
public static final String SOURCE_NAME = XceiverClientMetrics.class
.getSimpleName();
private @Metric MutableCounterLong pendingOps;
private @Metric MutableCounterLong totalOps;
private MutableCounterLong[] pendingOpsArray;
private MutableCounterLong[] opsArray;
private MutableRate[] containerOpsLatency;
private MetricsRegistry registry;
public XceiverClientMetrics() {
int numEnumEntries = ContainerProtos.Type.values().length;
this.registry = new MetricsRegistry(SOURCE_NAME);
this.pendingOpsArray = new MutableCounterLong[numEnumEntries];
this.opsArray = new MutableCounterLong[numEnumEntries];
this.containerOpsLatency = new MutableRate[numEnumEntries];
for (int i = 0; i < numEnumEntries; i++) {
pendingOpsArray[i] = registry.newCounter(
"numPending" + ContainerProtos.Type.forNumber(i + 1),
"number of pending" + ContainerProtos.Type.forNumber(i + 1) + " ops",
(long) 0);
opsArray[i] = registry
.newCounter("opCount" + ContainerProtos.Type.forNumber(i + 1),
"number of" + ContainerProtos.Type.forNumber(i + 1) + " ops",
(long) 0);
containerOpsLatency[i] = registry.newRate(
ContainerProtos.Type.forNumber(i + 1) + "Latency",
"latency of " + ContainerProtos.Type.forNumber(i + 1)
+ " ops");
}
}
public static XceiverClientMetrics create() {
DefaultMetricsSystem.initialize(SOURCE_NAME);
MetricsSystem ms = DefaultMetricsSystem.instance();
return ms.register(SOURCE_NAME, "Storage Container Client Metrics",
new XceiverClientMetrics());
}
public void incrPendingContainerOpsMetrics(ContainerProtos.Type type) {
pendingOps.incr();
totalOps.incr();
opsArray[type.ordinal()].incr();
pendingOpsArray[type.ordinal()].incr();
}
public void decrPendingContainerOpsMetrics(ContainerProtos.Type type) {
pendingOps.incr(-1);
pendingOpsArray[type.ordinal()].incr(-1);
}
public void addContainerOpsLatency(ContainerProtos.Type type,
long latencyNanos) {
containerOpsLatency[type.ordinal()].add(latencyNanos);
}
public long getContainerOpsMetrics(ContainerProtos.Type type) {
return pendingOpsArray[type.ordinal()].value();
}
@VisibleForTesting
public long getTotalOpCount() {
return totalOps.value();
}
@VisibleForTesting
public long getContainerOpCountMetrics(ContainerProtos.Type type) {
return opsArray[type.ordinal()].value();
}
public void unRegister() {
MetricsSystem ms = DefaultMetricsSystem.instance();
ms.unregisterSource(SOURCE_NAME);
}
}

367
hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientRatis.java
View File

@ -1,367 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdds.scm;
import java.io.IOException;
import java.security.cert.X509Certificate;
import java.util.Collection;
import java.util.List;
import java.util.Objects;
import java.util.OptionalLong;
import java.util.UUID;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.CompletionException;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicReference;
import java.util.stream.Collectors;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdds.HddsUtils;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandRequestProto;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandResponseProto;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.ratis.ContainerCommandRequestMessage;
import org.apache.hadoop.hdds.scm.client.HddsClientUtils;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
import org.apache.hadoop.hdds.security.x509.SecurityConfig;
import org.apache.hadoop.hdds.tracing.TracingUtil;
import org.apache.hadoop.util.Time;
import org.apache.hadoop.hdds.ratis.RatisHelper;
import org.apache.ratis.client.RaftClient;
import org.apache.ratis.grpc.GrpcTlsConfig;
import org.apache.ratis.proto.RaftProtos;
import org.apache.ratis.protocol.GroupMismatchException;
import org.apache.ratis.protocol.RaftClientReply;
import org.apache.ratis.protocol.RaftException;
import org.apache.ratis.retry.RetryPolicy;
import org.apache.ratis.rpc.RpcType;
import org.apache.ratis.rpc.SupportedRpcType;
import org.apache.ratis.thirdparty.com.google.protobuf.InvalidProtocolBufferException;
import org.apache.ratis.util.TimeDuration;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import io.opentracing.Scope;
import io.opentracing.util.GlobalTracer;
/**
* An abstract implementation of {@link XceiverClientSpi} using Ratis.
* The underlying RPC mechanism can be chosen via the constructor.
*/
public final class XceiverClientRatis extends XceiverClientSpi {
public static final Logger LOG =
LoggerFactory.getLogger(XceiverClientRatis.class);
public static XceiverClientRatis newXceiverClientRatis(
org.apache.hadoop.hdds.scm.pipeline.Pipeline pipeline,
Configuration ozoneConf) {
return newXceiverClientRatis(pipeline, ozoneConf, null);
}
public static XceiverClientRatis newXceiverClientRatis(
org.apache.hadoop.hdds.scm.pipeline.Pipeline pipeline,
Configuration ozoneConf, X509Certificate caCert) {
final String rpcType = ozoneConf
.get(ScmConfigKeys.DFS_CONTAINER_RATIS_RPC_TYPE_KEY,
ScmConfigKeys.DFS_CONTAINER_RATIS_RPC_TYPE_DEFAULT);
final TimeDuration clientRequestTimeout =
RatisHelper.getClientRequestTimeout(ozoneConf);
final int maxOutstandingRequests =
HddsClientUtils.getMaxOutstandingRequests(ozoneConf);
final RetryPolicy retryPolicy = RatisHelper.createRetryPolicy(ozoneConf);
final GrpcTlsConfig tlsConfig = RatisHelper.createTlsClientConfig(new
SecurityConfig(ozoneConf), caCert);
return new XceiverClientRatis(pipeline,
SupportedRpcType.valueOfIgnoreCase(rpcType), maxOutstandingRequests,
retryPolicy, tlsConfig, clientRequestTimeout);
}
private final Pipeline pipeline;
private final RpcType rpcType;
private final AtomicReference<RaftClient> client = new AtomicReference<>();
private final int maxOutstandingRequests;
private final RetryPolicy retryPolicy;
private final GrpcTlsConfig tlsConfig;
private final TimeDuration clientRequestTimeout;
// Map to track commit index at every server
private final ConcurrentHashMap<UUID, Long> commitInfoMap;
private XceiverClientMetrics metrics;
/**
* Constructs a client.
*/
private XceiverClientRatis(Pipeline pipeline, RpcType rpcType,
int maxOutStandingChunks, RetryPolicy retryPolicy,
GrpcTlsConfig tlsConfig, TimeDuration timeout) {
super();
this.pipeline = pipeline;
this.rpcType = rpcType;
this.maxOutstandingRequests = maxOutStandingChunks;
this.retryPolicy = retryPolicy;
commitInfoMap = new ConcurrentHashMap<>();
this.tlsConfig = tlsConfig;
this.clientRequestTimeout = timeout;
metrics = XceiverClientManager.getXceiverClientMetrics();
}
private void updateCommitInfosMap(
Collection<RaftProtos.CommitInfoProto> commitInfoProtos) {
// if the commitInfo map is empty, just update the commit indexes for each
// of the servers
if (commitInfoMap.isEmpty()) {
commitInfoProtos.forEach(proto -> commitInfoMap
.put(RatisHelper.toDatanodeId(proto.getServer()),
proto.getCommitIndex()));
// In case the commit is happening 2 way, just update the commitIndex
// for the servers which have been successfully updating the commit
// indexes. This is important because getReplicatedMinCommitIndex()
// should always return the min commit index out of the nodes which have
// been replicating data successfully.
} else {
commitInfoProtos.forEach(proto -> commitInfoMap
.computeIfPresent(RatisHelper.toDatanodeId(proto.getServer()),
(address, index) -> {
index = proto.getCommitIndex();
return index;
}));
}
}
/**
* Returns Ratis as pipeline Type.
*
* @return - Ratis
*/
@Override
public HddsProtos.ReplicationType getPipelineType() {
return HddsProtos.ReplicationType.RATIS;
}
@Override
public Pipeline getPipeline() {
return pipeline;
}
@Override
public void connect() throws Exception {
if (LOG.isDebugEnabled()) {
LOG.debug("Connecting to pipeline:{} datanode:{}", getPipeline().getId(),
RatisHelper.toRaftPeerId(pipeline.getFirstNode()));
}
// TODO : XceiverClient ratis should pass the config value of
// maxOutstandingRequests so as to set the upper bound on max no of async
// requests to be handled by raft client
if (!client.compareAndSet(null,
RatisHelper.newRaftClient(rpcType, getPipeline(), retryPolicy,
maxOutstandingRequests, tlsConfig, clientRequestTimeout))) {
throw new IllegalStateException("Client is already connected.");
}
}
@Override
public void connect(String encodedToken) throws Exception {
throw new UnsupportedOperationException("Block tokens are not " +
"implemented for Ratis clients.");
}
@Override
public void close() {
final RaftClient c = client.getAndSet(null);
if (c != null) {
closeRaftClient(c);
}
}
private void closeRaftClient(RaftClient raftClient) {
try {
raftClient.close();
} catch (IOException e) {
throw new IllegalStateException(e);
}
}
private RaftClient getClient() {
return Objects.requireNonNull(client.get(), "client is null");
}
@VisibleForTesting
public ConcurrentHashMap<UUID, Long> getCommitInfoMap() {
return commitInfoMap;
}
private CompletableFuture<RaftClientReply> sendRequestAsync(
ContainerCommandRequestProto request) {
try (Scope scope = GlobalTracer.get()
.buildSpan("XceiverClientRatis." + request.getCmdType().name())
.startActive(true)) {
final ContainerCommandRequestMessage message
= ContainerCommandRequestMessage.toMessage(
request, TracingUtil.exportCurrentSpan());
if (HddsUtils.isReadOnly(request)) {
if (LOG.isDebugEnabled()) {
LOG.debug("sendCommandAsync ReadOnly {}", message);
}
return getClient().sendReadOnlyAsync(message);
} else {
if (LOG.isDebugEnabled()) {
LOG.debug("sendCommandAsync {}", message);
}
return getClient().sendAsync(message);
}
}
}
// gets the minimum log index replicated to all servers
@Override
public long getReplicatedMinCommitIndex() {
OptionalLong minIndex =
commitInfoMap.values().parallelStream().mapToLong(v -> v).min();
return minIndex.isPresent() ? minIndex.getAsLong() : 0;
}
private void addDatanodetoReply(UUID address, XceiverClientReply reply) {
DatanodeDetails.Builder builder = DatanodeDetails.newBuilder();
builder.setUuid(address.toString());
reply.addDatanode(builder.build());
}
@Override
public XceiverClientReply watchForCommit(long index, long timeout)
throws InterruptedException, ExecutionException, TimeoutException,
IOException {
long commitIndex = getReplicatedMinCommitIndex();
XceiverClientReply clientReply = new XceiverClientReply(null);
if (commitIndex >= index) {
// return the min commit index till which the log has been replicated to
// all servers
clientReply.setLogIndex(commitIndex);
return clientReply;
}
if (LOG.isDebugEnabled()) {
LOG.debug("commit index : {} watch timeout : {}", index, timeout);
}
RaftClientReply reply;
try {
CompletableFuture<RaftClientReply> replyFuture = getClient()
.sendWatchAsync(index, RaftProtos.ReplicationLevel.ALL_COMMITTED);
replyFuture.get(timeout, TimeUnit.MILLISECONDS);
} catch (Exception e) {
Throwable t = HddsClientUtils.checkForException(e);
LOG.warn("3 way commit failed on pipeline {}", pipeline, e);
if (t instanceof GroupMismatchException) {
throw e;
}
reply = getClient()
.sendWatchAsync(index, RaftProtos.ReplicationLevel.MAJORITY_COMMITTED)
.get(timeout, TimeUnit.MILLISECONDS);
List<RaftProtos.CommitInfoProto> commitInfoProtoList =
reply.getCommitInfos().stream()
.filter(i -> i.getCommitIndex() < index)
.collect(Collectors.toList());
commitInfoProtoList.parallelStream().forEach(proto -> {
UUID address = RatisHelper.toDatanodeId(proto.getServer());
addDatanodetoReply(address, clientReply);
// since 3 way commit has failed, the updated map from now on will
// only store entries for those datanodes which have had successful
// replication.
commitInfoMap.remove(address);
LOG.info(
"Could not commit index {} on pipeline {} to all the nodes. " +
"Server {} has failed. Committed by majority.",
index, pipeline, address);
});
}
clientReply.setLogIndex(index);
return clientReply;
}
/**
* Sends a given command to server gets a waitable future back.
*
* @param request Request
* @return Response to the command
*/
@Override
public XceiverClientReply sendCommandAsync(
ContainerCommandRequestProto request) {
XceiverClientReply asyncReply = new XceiverClientReply(null);
long requestTime = Time.monotonicNowNanos();
CompletableFuture<RaftClientReply> raftClientReply =
sendRequestAsync(request);
metrics.incrPendingContainerOpsMetrics(request.getCmdType());
CompletableFuture<ContainerCommandResponseProto> containerCommandResponse =
raftClientReply.whenComplete((reply, e) -> {
if (LOG.isDebugEnabled()) {
LOG.debug("received reply {} for request: cmdType={} containerID={}"
+ " pipelineID={} traceID={} exception: {}", reply,
request.getCmdType(), request.getContainerID(),
request.getPipelineID(), request.getTraceID(), e);
}
metrics.decrPendingContainerOpsMetrics(request.getCmdType());
metrics.addContainerOpsLatency(request.getCmdType(),
Time.monotonicNowNanos() - requestTime);
}).thenApply(reply -> {
try {
if (!reply.isSuccess()) {
// in case of raft retry failure, the raft client is
// not able to connect to the leader hence the pipeline
// can not be used but this instance of RaftClient will close
// and refreshed again. In case the client cannot connect to
// leader, getClient call will fail.
// No need to set the failed Server ID here. Ozone client
// will directly exclude this pipeline in next allocate block
// to SCM as in this case, it is the raft client which is not
// able to connect to leader in the pipeline, though the
// pipeline can still be functional.
RaftException exception = reply.getException();
Preconditions.checkNotNull(exception, "Raft reply failure but " +
"no exception propagated.");
throw new CompletionException(exception);
}
ContainerCommandResponseProto response =
ContainerCommandResponseProto
.parseFrom(reply.getMessage().getContent());
UUID serverId = RatisHelper.toDatanodeId(reply.getReplierId());
if (response.getResult() == ContainerProtos.Result.SUCCESS) {
updateCommitInfosMap(reply.getCommitInfos());
}
asyncReply.setLogIndex(reply.getLogIndex());
addDatanodetoReply(serverId, asyncReply);
return response;
} catch (InvalidProtocolBufferException e) {
throw new CompletionException(e);
}
});
asyncReply.setResponse(containerCommandResponse);
return asyncReply;
}
}

495
hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/client/ContainerOperationClient.java
View File

@ -1,495 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdds.scm.client;
import com.google.common.base.Preconditions;
import org.apache.hadoop.hdds.scm.XceiverClientManager;
import org.apache.hadoop.hdds.scm.XceiverClientSpi;
import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline;
import org.apache.hadoop.hdds.scm.container.ContainerInfo;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
import org.apache.hadoop.hdds.scm.protocol.StorageContainerLocationProtocol;
import org.apache.hadoop.hdds.scm.storage.ContainerProtocolCalls;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
.ContainerDataProto;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
.ReadContainerResponseProto;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.protocol.proto
.StorageContainerLocationProtocolProtos.ObjectStageChangeRequestProto;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.List;
/**
* This class provides the client-facing APIs of container operations.
*/
public class ContainerOperationClient implements ScmClient {
private static final Logger LOG =
LoggerFactory.getLogger(ContainerOperationClient.class);
private static long containerSizeB = -1;
private final StorageContainerLocationProtocol
storageContainerLocationClient;
private final XceiverClientManager xceiverClientManager;
public ContainerOperationClient(
StorageContainerLocationProtocol
storageContainerLocationClient,
XceiverClientManager xceiverClientManager) {
this.storageContainerLocationClient = storageContainerLocationClient;
this.xceiverClientManager = xceiverClientManager;
}
/**
* Return the capacity of containers. The current assumption is that all
* containers have the same capacity. Therefore one static is sufficient for
* any container.
* @return The capacity of one container in number of bytes.
*/
public static long getContainerSizeB() {
return containerSizeB;
}
/**
* Set the capacity of container. Should be exactly once on system start.
* @param size Capacity of one container in number of bytes.
*/
public static void setContainerSizeB(long size) {
containerSizeB = size;
}
@Override
public ContainerWithPipeline createContainer(String owner)
throws IOException {
XceiverClientSpi client = null;
try {
ContainerWithPipeline containerWithPipeline =
storageContainerLocationClient.allocateContainer(
xceiverClientManager.getType(),
xceiverClientManager.getFactor(), owner);
Pipeline pipeline = containerWithPipeline.getPipeline();
client = xceiverClientManager.acquireClient(pipeline);
Preconditions.checkState(pipeline.isOpen(), String
.format("Unexpected state=%s for pipeline=%s, expected state=%s",
pipeline.getPipelineState(), pipeline.getId(),
Pipeline.PipelineState.OPEN));
createContainer(client,
containerWithPipeline.getContainerInfo().getContainerID());
return containerWithPipeline;
} finally {
if (client != null) {
xceiverClientManager.releaseClient(client, false);
}
}
}
/**
* Create a container over pipeline specified by the SCM.
*
* @param client - Client to communicate with Datanodes.
* @param containerId - Container ID.
* @throws IOException
*/
public void createContainer(XceiverClientSpi client,
long containerId) throws IOException {
ContainerProtocolCalls.createContainer(client, containerId, null);
// Let us log this info after we let SCM know that we have completed the
// creation state.
if (LOG.isDebugEnabled()) {
LOG.debug("Created container " + containerId
+ " machines:" + client.getPipeline().getNodes());
}
}
/**
* Creates a pipeline over the machines choosen by the SCM.
*
* @param client - Client
* @param pipeline - pipeline to be createdon Datanodes.
* @throws IOException
*/
private void createPipeline(XceiverClientSpi client, Pipeline pipeline)
throws IOException {
Preconditions.checkNotNull(pipeline.getId(), "Pipeline " +
"name cannot be null when client create flag is set.");
// Pipeline creation is a three step process.
//
// 1. Notify SCM that this client is doing a create pipeline on
// datanodes.
//
// 2. Talk to Datanodes to create the pipeline.
//
// 3. update SCM that pipeline creation was successful.
// TODO: this has not been fully implemented on server side
// SCMClientProtocolServer#notifyObjectStageChange
// TODO: when implement the pipeline state machine, change
// the pipeline name (string) to pipeline id (long)
//storageContainerLocationClient.notifyObjectStageChange(
// ObjectStageChangeRequestProto.Type.pipeline,
// pipeline.getPipelineName(),
// ObjectStageChangeRequestProto.Op.create,
// ObjectStageChangeRequestProto.Stage.begin);
// client.createPipeline();
// TODO: Use PipelineManager to createPipeline
//storageContainerLocationClient.notifyObjectStageChange(
// ObjectStageChangeRequestProto.Type.pipeline,
// pipeline.getPipelineName(),
// ObjectStageChangeRequestProto.Op.create,
// ObjectStageChangeRequestProto.Stage.complete);
// TODO : Should we change the state on the client side ??
// That makes sense, but it is not needed for the client to work.
if (LOG.isDebugEnabled()) {
LOG.debug("Pipeline creation successful. Pipeline: {}",
pipeline.toString());
}
}
@Override
public ContainerWithPipeline createContainer(HddsProtos.ReplicationType type,
HddsProtos.ReplicationFactor factor, String owner) throws IOException {
XceiverClientSpi client = null;
try {
// allocate container on SCM.
ContainerWithPipeline containerWithPipeline =
storageContainerLocationClient.allocateContainer(type, factor,
owner);
Pipeline pipeline = containerWithPipeline.getPipeline();
client = xceiverClientManager.acquireClient(pipeline);
// connect to pipeline leader and allocate container on leader datanode.
client = xceiverClientManager.acquireClient(pipeline);
createContainer(client,
containerWithPipeline.getContainerInfo().getContainerID());
return containerWithPipeline;
} finally {
if (client != null) {
xceiverClientManager.releaseClient(client, false);
}
}
}
/**
* Returns a set of Nodes that meet a query criteria.
*
* @param nodeStatuses - Criteria that we want the node to have.
* @param queryScope - Query scope - Cluster or pool.
* @param poolName - if it is pool, a pool name is required.
* @return A set of nodes that meet the requested criteria.
* @throws IOException
*/
@Override
public List<HddsProtos.Node> queryNode(HddsProtos.NodeState
nodeStatuses, HddsProtos.QueryScope queryScope, String poolName)
throws IOException {
return storageContainerLocationClient.queryNode(nodeStatuses, queryScope,
poolName);
}
/**
* Creates a specified replication pipeline.
*/
@Override
public Pipeline createReplicationPipeline(HddsProtos.ReplicationType type,
HddsProtos.ReplicationFactor factor, HddsProtos.NodePool nodePool)
throws IOException {
return storageContainerLocationClient.createReplicationPipeline(type,
factor, nodePool);
}
@Override
public List<Pipeline> listPipelines() throws IOException {
return storageContainerLocationClient.listPipelines();
}
@Override
public void activatePipeline(HddsProtos.PipelineID pipelineID)
throws IOException {
storageContainerLocationClient.activatePipeline(pipelineID);
}
@Override
public void deactivatePipeline(HddsProtos.PipelineID pipelineID)
throws IOException {
storageContainerLocationClient.deactivatePipeline(pipelineID);
}
@Override
public void closePipeline(HddsProtos.PipelineID pipelineID)
throws IOException {
storageContainerLocationClient.closePipeline(pipelineID);
}
@Override
public void close() {
try {
xceiverClientManager.close();
} catch (Exception ex) {
LOG.error("Can't close " + this.getClass().getSimpleName(), ex);
}
}
/**
* Deletes an existing container.
*
* @param containerId - ID of the container.
* @param pipeline - Pipeline that represents the container.
* @param force - true to forcibly delete the container.
* @throws IOException
*/
@Override
public void deleteContainer(long containerId, Pipeline pipeline,
boolean force) throws IOException {
XceiverClientSpi client = null;
try {
client = xceiverClientManager.acquireClient(pipeline);
ContainerProtocolCalls
.deleteContainer(client, containerId, force, null);
storageContainerLocationClient
.deleteContainer(containerId);
if (LOG.isDebugEnabled()) {
LOG.debug("Deleted container {}, machines: {} ", containerId,
pipeline.getNodes());
}
} finally {
if (client != null) {
xceiverClientManager.releaseClient(client, false);
}
}
}
/**
* Delete the container, this will release any resource it uses.
* @param containerID - containerID.
* @param force - True to forcibly delete the container.
* @throws IOException
*/
@Override
public void deleteContainer(long containerID, boolean force)
throws IOException {
ContainerWithPipeline info = getContainerWithPipeline(containerID);
deleteContainer(containerID, info.getPipeline(), force);
}
@Override
public List<ContainerInfo> listContainer(long startContainerID,
int count) throws IOException {
return storageContainerLocationClient.listContainer(
startContainerID, count);
}
/**
* Get meta data from an existing container.
*
* @param containerID - ID of the container.
* @param pipeline - Pipeline where the container is located.
* @return ContainerInfo
* @throws IOException
*/
@Override
public ContainerDataProto readContainer(long containerID,
Pipeline pipeline) throws IOException {
XceiverClientSpi client = null;
try {
client = xceiverClientManager.acquireClient(pipeline);
ReadContainerResponseProto response =
ContainerProtocolCalls.readContainer(client, containerID, null);
if (LOG.isDebugEnabled()) {
LOG.debug("Read container {}, machines: {} ", containerID,
pipeline.getNodes());
}
return response.getContainerData();
} finally {
if (client != null) {
xceiverClientManager.releaseClient(client, false);
}
}
}
/**
* Get meta data from an existing container.
* @param containerID - ID of the container.
* @return ContainerInfo - a message of protobuf which has basic info
* of a container.
* @throws IOException
*/
@Override
public ContainerDataProto readContainer(long containerID) throws IOException {
ContainerWithPipeline info = getContainerWithPipeline(containerID);
return readContainer(containerID, info.getPipeline());
}
/**
* Given an id, return the pipeline associated with the container.
* @param containerId - String Container ID
* @return Pipeline of the existing container, corresponding to the given id.
* @throws IOException
*/
@Override
public ContainerInfo getContainer(long containerId) throws
IOException {
return storageContainerLocationClient.getContainer(containerId);
}
/**
* Gets a container by Name -- Throws if the container does not exist.
*
* @param containerId - Container ID
* @return ContainerWithPipeline
* @throws IOException
*/
@Override
public ContainerWithPipeline getContainerWithPipeline(long containerId)
throws IOException {
return storageContainerLocationClient.getContainerWithPipeline(containerId);
}
/**
* Close a container.
*
* @param pipeline the container to be closed.
* @throws IOException
*/
@Override
public void closeContainer(long containerId, Pipeline pipeline)
throws IOException {
XceiverClientSpi client = null;
try {
if (LOG.isDebugEnabled()) {
LOG.debug("Close container {}", pipeline);
}
/*
TODO: two orders here, revisit this later:
1. close on SCM first, then on data node
2. close on data node first, then on SCM
with 1: if client failed after closing on SCM, then there is a
container SCM thinks as closed, but is actually open. Then SCM will no
longer allocate block to it, which is fine. But SCM may later try to
replicate this "closed" container, which I'm not sure is safe.
with 2: if client failed after close on datanode, then there is a
container SCM thinks as open, but is actually closed. Then SCM will still
try to allocate block to it. Which will fail when actually doing the
write. No more data can be written, but at least the correctness and
consistency of existing data will maintain.
For now, take the #2 way.
*/
// Actually close the container on Datanode
client = xceiverClientManager.acquireClient(pipeline);
storageContainerLocationClient.notifyObjectStageChange(
ObjectStageChangeRequestProto.Type.container,
containerId,
ObjectStageChangeRequestProto.Op.close,
ObjectStageChangeRequestProto.Stage.begin);
ContainerProtocolCalls.closeContainer(client, containerId,
null);
// Notify SCM to close the container
storageContainerLocationClient.notifyObjectStageChange(
ObjectStageChangeRequestProto.Type.container,
containerId,
ObjectStageChangeRequestProto.Op.close,
ObjectStageChangeRequestProto.Stage.complete);
} finally {
if (client != null) {
xceiverClientManager.releaseClient(client, false);
}
}
}
/**
* Close a container.
*
* @throws IOException
*/
@Override
public void closeContainer(long containerId)
throws IOException {
ContainerWithPipeline info = getContainerWithPipeline(containerId);
Pipeline pipeline = info.getPipeline();
closeContainer(containerId, pipeline);
}
/**
* Get the the current usage information.
* @param containerID - ID of the container.
* @return the size of the given container.
* @throws IOException
*/
@Override
public long getContainerSize(long containerID) throws IOException {
// TODO : Fix this, it currently returns the capacity
// but not the current usage.
long size = getContainerSizeB();
if (size == -1) {
throw new IOException("Container size unknown!");
}
return size;
}
/**
* Check if SCM is in safe mode.
*
* @return Returns true if SCM is in safe mode else returns false.
* @throws IOException
*/
public boolean inSafeMode() throws IOException {
return storageContainerLocationClient.inSafeMode();
}
/**
* Force SCM out of safe mode.
*
* @return returns true if operation is successful.
* @throws IOException
*/
public boolean forceExitSafeMode() throws IOException {
return storageContainerLocationClient.forceExitSafeMode();
}
@Override
public void startReplicationManager() throws IOException {
storageContainerLocationClient.startReplicationManager();
}
@Override
public void stopReplicationManager() throws IOException {
storageContainerLocationClient.stopReplicationManager();
}
@Override
public boolean getReplicationManagerStatus() throws IOException {
return storageContainerLocationClient.getReplicationManagerStatus();
}
}

350
hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/client/HddsClientUtils.java
View File

@ -1,350 +0,0 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdds.scm.client;
import com.google.common.base.Preconditions;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdds.HddsUtils;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.SCMSecurityProtocol;
import org.apache.hadoop.hdds.protocolPB.SCMSecurityProtocolClientSideTranslatorPB;
import org.apache.hadoop.hdds.protocolPB.SCMSecurityProtocolPB;
import org.apache.hadoop.hdds.scm.XceiverClientManager.ScmClientConfig;
import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException;
import org.apache.hadoop.hdds.scm.protocol.ScmBlockLocationProtocol;
import org.apache.hadoop.hdds.scm.protocolPB.ScmBlockLocationProtocolPB;
import org.apache.hadoop.io.retry.RetryPolicies;
import org.apache.hadoop.io.retry.RetryPolicy;
import org.apache.hadoop.ipc.Client;
import org.apache.hadoop.ipc.ProtobufRpcEngine;
import org.apache.hadoop.ipc.RPC;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.ratis.protocol.AlreadyClosedException;
import org.apache.ratis.protocol.GroupMismatchException;
import org.apache.ratis.protocol.NotReplicatedException;
import org.apache.ratis.protocol.RaftRetryFailureException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.text.ParseException;
import java.time.Instant;
import java.time.ZoneId;
import java.time.ZonedDateTime;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
/**
* Utility methods for Ozone and Container Clients.
*
* The methods to retrieve SCM service endpoints assume there is a single
* SCM service instance. This will change when we switch to replicated service
* instances for redundancy.
*/
@InterfaceAudience.Public
@InterfaceStability.Unstable
public final class HddsClientUtils {
private static final Logger LOG = LoggerFactory.getLogger(
HddsClientUtils.class);
private static final int NO_PORT = -1;
private HddsClientUtils() {
}
private static final List<Class<? extends Exception>> EXCEPTION_LIST =
new ArrayList<Class<? extends Exception>>() {{
add(TimeoutException.class);
add(StorageContainerException.class);
add(RaftRetryFailureException.class);
add(AlreadyClosedException.class);
add(GroupMismatchException.class);
// Not Replicated Exception will be thrown if watch For commit
// does not succeed
add(NotReplicatedException.class);
}};
/**
* Date format that used in ozone. Here the format is thread safe to use.
*/
private static final ThreadLocal<DateTimeFormatter> DATE_FORMAT =
ThreadLocal.withInitial(() -> {
DateTimeFormatter format =
DateTimeFormatter.ofPattern(OzoneConsts.OZONE_DATE_FORMAT);
return format.withZone(ZoneId.of(OzoneConsts.OZONE_TIME_ZONE));
});
/**
* Convert time in millisecond to a human readable format required in ozone.
* @return a human readable string for the input time
*/
public static String formatDateTime(long millis) {
ZonedDateTime dateTime = ZonedDateTime.ofInstant(
Instant.ofEpochMilli(millis), DATE_FORMAT.get().getZone());
return DATE_FORMAT.get().format(dateTime);
}
/**
* Convert time in ozone date format to millisecond.
* @return time in milliseconds
*/
public static long formatDateTime(String date) throws ParseException {
Preconditions.checkNotNull(date, "Date string should not be null.");
return ZonedDateTime.parse(date, DATE_FORMAT.get())
.toInstant().toEpochMilli();
}
/**
* verifies that bucket name / volume name is a valid DNS name.
*
* @param resName Bucket or volume Name to be validated
*
* @throws IllegalArgumentException
*/
public static void verifyResourceName(String resName)
throws IllegalArgumentException {
if (resName == null) {
throw new IllegalArgumentException("Bucket or Volume name is null");
}
if (resName.length() < OzoneConsts.OZONE_MIN_BUCKET_NAME_LENGTH ||
resName.length() > OzoneConsts.OZONE_MAX_BUCKET_NAME_LENGTH) {
throw new IllegalArgumentException(
"Bucket or Volume length is illegal, "
+ "valid length is 3-63 characters");
}
if (resName.charAt(0) == '.' || resName.charAt(0) == '-') {
throw new IllegalArgumentException(
"Bucket or Volume name cannot start with a period or dash");
}
if (resName.charAt(resName.length() - 1) == '.' ||
resName.charAt(resName.length() - 1) == '-') {
throw new IllegalArgumentException("Bucket or Volume name "
+ "cannot end with a period or dash");
}
boolean isIPv4 = true;
char prev = (char) 0;
for (int index = 0; index < resName.length(); index++) {
char currChar = resName.charAt(index);
if (currChar != '.') {
isIPv4 = ((currChar >= '0') && (currChar <= '9')) && isIPv4;
}
if (currChar > 'A' && currChar < 'Z') {
throw new IllegalArgumentException(
"Bucket or Volume name does not support uppercase characters");
}
if (currChar != '.' && currChar != '-') {
if (currChar < '0' || (currChar > '9' && currChar < 'a') ||
currChar > 'z') {
throw new IllegalArgumentException("Bucket or Volume name has an " +
"unsupported character : " +
currChar);
}
}
if (prev == '.' && currChar == '.') {
throw new IllegalArgumentException("Bucket or Volume name should not " +
"have two contiguous periods");
}
if (prev == '-' && currChar == '.') {
throw new IllegalArgumentException(
"Bucket or Volume name should not have period after dash");
}
if (prev == '.' && currChar == '-') {
throw new IllegalArgumentException(
"Bucket or Volume name should not have dash after period");
}
prev = currChar;
}
if (isIPv4) {
throw new IllegalArgumentException(
"Bucket or Volume name cannot be an IPv4 address or all numeric");
}
}
/**
* verifies that bucket / volume name is a valid DNS name.
*
* @param resourceNames Array of bucket / volume names to be verified.
*/
public static void verifyResourceName(String... resourceNames) {
for (String resourceName : resourceNames) {
HddsClientUtils.verifyResourceName(resourceName);
}
}
/**
* Checks that object parameters passed as reference is not null.
*
* @param references Array of object references to be checked.
* @param <T>
*/
public static <T> void checkNotNull(T... references) {
for (T ref: references) {
Preconditions.checkNotNull(ref);
}
}
/**
* Returns the cache value to be used for list calls.
* @param conf Configuration object
* @return list cache size
*/
public static int getListCacheSize(Configuration conf) {
return conf.getInt(OzoneConfigKeys.OZONE_CLIENT_LIST_CACHE_SIZE,
OzoneConfigKeys.OZONE_CLIENT_LIST_CACHE_SIZE_DEFAULT);
}
/**
* @return a default instance of {@link CloseableHttpClient}.
*/
public static CloseableHttpClient newHttpClient() {
return HddsClientUtils.newHttpClient(new Configuration());
}
/**
* Returns a {@link CloseableHttpClient} configured by given configuration.
* If conf is null, returns a default instance.
*
* @param conf configuration
* @return a {@link CloseableHttpClient} instance.
*/
public static CloseableHttpClient newHttpClient(Configuration conf) {
long socketTimeout = OzoneConfigKeys
.OZONE_CLIENT_SOCKET_TIMEOUT_DEFAULT;
long connectionTimeout = OzoneConfigKeys
.OZONE_CLIENT_CONNECTION_TIMEOUT_DEFAULT;
if (conf != null) {
socketTimeout = conf.getTimeDuration(
OzoneConfigKeys.OZONE_CLIENT_SOCKET_TIMEOUT,
OzoneConfigKeys.OZONE_CLIENT_SOCKET_TIMEOUT_DEFAULT,
TimeUnit.MILLISECONDS);
connectionTimeout = conf.getTimeDuration(
OzoneConfigKeys.OZONE_CLIENT_CONNECTION_TIMEOUT,
OzoneConfigKeys.OZONE_CLIENT_CONNECTION_TIMEOUT_DEFAULT,
TimeUnit.MILLISECONDS);
}
CloseableHttpClient client = HttpClients.custom()
.setDefaultRequestConfig(
RequestConfig.custom()
.setSocketTimeout(Math.toIntExact(socketTimeout))
.setConnectTimeout(Math.toIntExact(connectionTimeout))
.build())
.build();
return client;
}
/**
* Returns the maximum no of outstanding async requests to be handled by
* Standalone and Ratis client.
*/
public static int getMaxOutstandingRequests(Configuration config) {
return OzoneConfiguration.of(config)
.getObject(ScmClientConfig.class)
.getMaxOutstandingRequests();
}
/**
* Create a scm block client, used by putKey() and getKey().
*
* @return {@link ScmBlockLocationProtocol}
* @throws IOException
*/
public static SCMSecurityProtocol getScmSecurityClient(
OzoneConfiguration conf, UserGroupInformation ugi) throws IOException {
RPC.setProtocolEngine(conf, SCMSecurityProtocolPB.class,
ProtobufRpcEngine.class);
long scmVersion =
RPC.getProtocolVersion(ScmBlockLocationProtocolPB.class);
InetSocketAddress scmSecurityProtoAdd =
HddsUtils.getScmAddressForSecurityProtocol(conf);
SCMSecurityProtocolClientSideTranslatorPB scmSecurityClient =
new SCMSecurityProtocolClientSideTranslatorPB(
RPC.getProxy(SCMSecurityProtocolPB.class, scmVersion,
scmSecurityProtoAdd, ugi, conf,
NetUtils.getDefaultSocketFactory(conf),
Client.getRpcTimeout(conf)));
return scmSecurityClient;
}
public static Throwable checkForException(Exception e) {
Throwable t = e;
while (t != null) {
for (Class<? extends Exception> cls : getExceptionList()) {
if (cls.isInstance(t)) {
return t;
}
}
t = t.getCause();
}
return t;
}
public static RetryPolicy createRetryPolicy(int maxRetryCount,
long retryInterval) {
// retry with fixed sleep between retries
return RetryPolicies.retryUpToMaximumCountWithFixedSleep(
maxRetryCount, retryInterval, TimeUnit.MILLISECONDS);
}
public static Map<Class<? extends Throwable>,
RetryPolicy> getRetryPolicyByException(int maxRetryCount,
long retryInterval) {
Map<Class<? extends Throwable>, RetryPolicy> policyMap = new HashMap<>();
for (Class<? extends Exception> ex : EXCEPTION_LIST) {
if (ex == TimeoutException.class
|| ex == RaftRetryFailureException.class) {
// retry without sleep
policyMap.put(ex, createRetryPolicy(maxRetryCount, 0));
} else {
// retry with fixed sleep between retries
policyMap.put(ex, createRetryPolicy(maxRetryCount, retryInterval));
}
}
// Default retry policy
policyMap
.put(Exception.class, createRetryPolicy(maxRetryCount, retryInterval));
return policyMap;
}
public static List<Class<? extends Exception>> getExceptionList() {
return EXCEPTION_LIST;
}
}

23
hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/client/package-info.java
View File

@ -1,23 +0,0 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdds.scm.client;
/**
* Client facing classes for the container operations.
*/

23
hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/package-info.java
View File

@ -1,23 +0,0 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdds.scm;
/**
* Classes for different type of container service client.
*/

388
hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockInputStream.java
View File

@ -1,388 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdds.scm.storage;
import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
import org.apache.hadoop.hdds.security.token.OzoneBlockTokenIdentifier;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.fs.Seekable;
import org.apache.hadoop.hdds.scm.XceiverClientManager;
import org.apache.hadoop.hdds.scm.XceiverClientSpi;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ChunkInfo;
import org.apache.hadoop.hdds.client.BlockID;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.DatanodeBlockID;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.GetBlockResponseProto;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.EOFException;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
/**
* An {@link InputStream} called from KeyInputStream to read a block from the
* container.
* This class encapsulates all state management for iterating
* through the sequence of chunks through {@link ChunkInputStream}.
*/
public class BlockInputStream extends InputStream implements Seekable {
private static final Logger LOG =
LoggerFactory.getLogger(BlockInputStream.class);
private static final int EOF = -1;
private final BlockID blockID;
private final long length;
private Pipeline pipeline;
private final Token<OzoneBlockTokenIdentifier> token;
private final boolean verifyChecksum;
private XceiverClientManager xceiverClientManager;
private XceiverClientSpi xceiverClient;
private boolean initialized = false;
// List of ChunkInputStreams, one for each chunk in the block
private List<ChunkInputStream> chunkStreams;
// chunkOffsets[i] stores the index of the first data byte in
// chunkStream i w.r.t the block data.
// Let’s say we have chunk size as 40 bytes. And let's say the parent
// block stores data from index 200 and has length 400.
// The first 40 bytes of this block will be stored in chunk[0], next 40 in
// chunk[1] and so on. But since the chunkOffsets are w.r.t the block only
// and not the key, the values in chunkOffsets will be [0, 40, 80,....].
private long[] chunkOffsets = null;
// Index of the chunkStream corresponding to the current position of the
// BlockInputStream i.e offset of the data to be read next from this block
private int chunkIndex;
// Position of the BlockInputStream is maintainted by this variable till
// the stream is initialized. This position is w.r.t to the block only and
// not the key.
// For the above example, if we seek to position 240 before the stream is
// initialized, then value of blockPosition will be set to 40.
// Once, the stream is initialized, the position of the stream
// will be determined by the current chunkStream and its position.
private long blockPosition = 0;
// Tracks the chunkIndex corresponding to the last blockPosition so that it
// can be reset if a new position is seeked.
private int chunkIndexOfPrevPosition;
public BlockInputStream(BlockID blockId, long blockLen, Pipeline pipeline,
Token<OzoneBlockTokenIdentifier> token, boolean verifyChecksum,
XceiverClientManager xceiverClientManager) {
this.blockID = blockId;
this.length = blockLen;
this.pipeline = pipeline;
this.token = token;
this.verifyChecksum = verifyChecksum;
this.xceiverClientManager = xceiverClientManager;
}
/**
* Initialize the BlockInputStream. Get the BlockData (list of chunks) from
* the Container and create the ChunkInputStreams for each Chunk in the Block.
*/
public synchronized void initialize() throws IOException {
// Pre-check that the stream has not been intialized already
if (initialized) {
return;
}
List<ChunkInfo> chunks = getChunkInfos();
if (chunks != null && !chunks.isEmpty()) {
// For each chunk in the block, create a ChunkInputStream and compute
// its chunkOffset
this.chunkOffsets = new long[chunks.size()];
long tempOffset = 0;
this.chunkStreams = new ArrayList<>(chunks.size());
for (int i = 0; i < chunks.size(); i++) {
addStream(chunks.get(i));
chunkOffsets[i] = tempOffset;
tempOffset += chunks.get(i).getLen();
}
initialized = true;
this.chunkIndex = 0;
if (blockPosition > 0) {
// Stream was seeked to blockPosition before initialization. Seek to the
// blockPosition now.
seek(blockPosition);
}
}
}
/**
* Send RPC call to get the block info from the container.
* @return List of chunks in this block.
*/
protected List<ChunkInfo> getChunkInfos() throws IOException {
// irrespective of the container state, we will always read via Standalone
// protocol.
if (pipeline.getType() != HddsProtos.ReplicationType.STAND_ALONE) {
pipeline = Pipeline.newBuilder(pipeline)
.setType(HddsProtos.ReplicationType.STAND_ALONE).build();
}
xceiverClient = xceiverClientManager.acquireClientForReadData(pipeline);
boolean success = false;
List<ChunkInfo> chunks;
try {
if (LOG.isDebugEnabled()) {
LOG.debug("Initializing BlockInputStream for get key to access {}",
blockID.getContainerID());
}
if (token != null) {
UserGroupInformation.getCurrentUser().addToken(token);
}
DatanodeBlockID datanodeBlockID = blockID
.getDatanodeBlockIDProtobuf();
GetBlockResponseProto response = ContainerProtocolCalls
.getBlock(xceiverClient, datanodeBlockID);
chunks = response.getBlockData().getChunksList();
success = true;
} finally {
if (!success) {
xceiverClientManager.releaseClientForReadData(xceiverClient, false);
}
}
return chunks;
}
/**
* Append another ChunkInputStream to the end of the list. Note that the
* ChunkInputStream is only created here. The chunk will be read from the
* Datanode only when a read operation is performed on for that chunk.
*/
protected synchronized void addStream(ChunkInfo chunkInfo) {
chunkStreams.add(new ChunkInputStream(chunkInfo, blockID,
xceiverClient, verifyChecksum));
}
public synchronized long getRemaining() throws IOException {
return length - getPos();
}
/**
* {@inheritDoc}
*/
@Override
public synchronized int read() throws IOException {
byte[] buf = new byte[1];
if (read(buf, 0, 1) == EOF) {
return EOF;
}
return Byte.toUnsignedInt(buf[0]);
}
/**
* {@inheritDoc}
*/
@Override
public synchronized int read(byte[] b, int off, int len) throws IOException {
if (b == null) {
throw new NullPointerException();
}
if (off < 0 || len < 0 || len > b.length - off) {
throw new IndexOutOfBoundsException();
}
if (len == 0) {
return 0;
}
if (!initialized) {
initialize();
}
checkOpen();
int totalReadLen = 0;
while (len > 0) {
// if we are at the last chunk and have read the entire chunk, return
if (chunkStreams.size() == 0 ||
(chunkStreams.size() - 1 <= chunkIndex &&
chunkStreams.get(chunkIndex)
.getRemaining() == 0)) {
return totalReadLen == 0 ? EOF : totalReadLen;
}
// Get the current chunkStream and read data from it
ChunkInputStream current = chunkStreams.get(chunkIndex);
int numBytesToRead = Math.min(len, (int)current.getRemaining());
int numBytesRead = current.read(b, off, numBytesToRead);
if (numBytesRead != numBytesToRead) {
// This implies that there is either data loss or corruption in the
// chunk entries. Even EOF in the current stream would be covered in
// this case.
throw new IOException(String.format(
"Inconsistent read for chunkName=%s length=%d numBytesRead=%d",
current.getChunkName(), current.getLength(), numBytesRead));
}
totalReadLen += numBytesRead;
off += numBytesRead;
len -= numBytesRead;
if (current.getRemaining() <= 0 &&
((chunkIndex + 1) < chunkStreams.size())) {
chunkIndex += 1;
}
}
return totalReadLen;
}
/**
* Seeks the BlockInputStream to the specified position. If the stream is
* not initialized, save the seeked position via blockPosition. Otherwise,
* update the position in 2 steps:
* 1. Updating the chunkIndex to the chunkStream corresponding to the
* seeked position.
* 2. Seek the corresponding chunkStream to the adjusted position.
*
* Let’s say we have chunk size as 40 bytes. And let's say the parent block
* stores data from index 200 and has length 400. If the key was seeked to
* position 90, then this block will be seeked to position 90.
* When seek(90) is called on this blockStream, then
* 1. chunkIndex will be set to 2 (as indices 80 - 120 reside in chunk[2]).
* 2. chunkStream[2] will be seeked to position 10
* (= 90 - chunkOffset[2] (= 80)).
*/
@Override
public synchronized void seek(long pos) throws IOException {
if (!initialized) {
// Stream has not been initialized yet. Save the position so that it
// can be seeked when the stream is initialized.
blockPosition = pos;
return;
}
checkOpen();
if (pos < 0 || pos >= length) {
if (pos == 0) {
// It is possible for length and pos to be zero in which case
// seek should return instead of throwing exception
return;
}
throw new EOFException(
"EOF encountered at pos: " + pos + " for block: " + blockID);
}
if (chunkIndex >= chunkStreams.size()) {
chunkIndex = Arrays.binarySearch(chunkOffsets, pos);
} else if (pos < chunkOffsets[chunkIndex]) {
chunkIndex =
Arrays.binarySearch(chunkOffsets, 0, chunkIndex, pos);
} else if (pos >= chunkOffsets[chunkIndex] + chunkStreams
.get(chunkIndex).getLength()) {
chunkIndex = Arrays.binarySearch(chunkOffsets,
chunkIndex + 1, chunkStreams.size(), pos);
}
if (chunkIndex < 0) {
// Binary search returns -insertionPoint - 1 if element is not present
// in the array. insertionPoint is the point at which element would be
// inserted in the sorted array. We need to adjust the chunkIndex
// accordingly so that chunkIndex = insertionPoint - 1
chunkIndex = -chunkIndex - 2;
}
// Reset the previous chunkStream's position
chunkStreams.get(chunkIndexOfPrevPosition).resetPosition();
// seek to the proper offset in the ChunkInputStream
chunkStreams.get(chunkIndex).seek(pos - chunkOffsets[chunkIndex]);
chunkIndexOfPrevPosition = chunkIndex;
}
@Override
public synchronized long getPos() throws IOException {
if (length == 0) {
return 0;
}
if (!initialized) {
// The stream is not initialized yet. Return the blockPosition
return blockPosition;
} else {
return chunkOffsets[chunkIndex] + chunkStreams.get(chunkIndex).getPos();
}
}
@Override
public boolean seekToNewSource(long targetPos) throws IOException {
return false;
}
@Override
public synchronized void close() {
if (xceiverClientManager != null && xceiverClient != null) {
xceiverClientManager.releaseClient(xceiverClient, false);
xceiverClientManager = null;
xceiverClient = null;
}
}
public synchronized void resetPosition() {
this.blockPosition = 0;
}
/**
* Checks if the stream is open. If not, throw an exception.
*
* @throws IOException if stream is closed
*/
protected synchronized void checkOpen() throws IOException {
if (xceiverClient == null) {
throw new IOException("BlockInputStream has been closed.");
}
}
public BlockID getBlockID() {
return blockID;
}
public long getLength() {
return length;
}
@VisibleForTesting
synchronized int getChunkIndex() {
return chunkIndex;
}
@VisibleForTesting
synchronized long getBlockPosition() {
return blockPosition;
}
@VisibleForTesting
synchronized List<ChunkInputStream> getChunkStreams() {
return chunkStreams;
}
}

640
hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockOutputStream.java
View File

@ -1,640 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdds.scm.storage;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
import org.apache.hadoop.hdds.scm.XceiverClientReply;
import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
import org.apache.hadoop.ozone.common.Checksum;
import org.apache.hadoop.ozone.common.ChecksumData;
import org.apache.hadoop.ozone.common.OzoneChecksumException;
import org.apache.ratis.thirdparty.com.google.protobuf.ByteString;
import org.apache.hadoop.hdds.scm.XceiverClientManager;
import org.apache.hadoop.hdds.scm.XceiverClientSpi;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ChecksumType;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ChunkInfo;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.BlockData;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.KeyValue;
import org.apache.hadoop.hdds.client.BlockID;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.io.OutputStream;
import java.nio.ByteBuffer;
import java.util.List;
import java.util.ArrayList;
import java.util.Map;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.CompletionException;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Executors;
import java.util.concurrent.atomic.AtomicReference;
import static org.apache.hadoop.hdds.scm.storage.ContainerProtocolCalls
.putBlockAsync;
import static org.apache.hadoop.hdds.scm.storage.ContainerProtocolCalls
.writeChunkAsync;
/**
* An {@link OutputStream} used by the REST service in combination with the
* SCMClient to write the value of a key to a sequence
* of container chunks. Writes are buffered locally and periodically written to
* the container as a new chunk. In order to preserve the semantics that
* replacement of a pre-existing key is atomic, each instance of the stream has
* an internal unique identifier. This unique identifier and a monotonically
* increasing chunk index form a composite key that is used as the chunk name.
* After all data is written, a putKey call creates or updates the corresponding
* container key, and this call includes the full list of chunks that make up
* the key data. The list of chunks is updated all at once. Therefore, a
* concurrent reader never can see an intermediate state in which different
* chunks of data from different versions of the key data are interleaved.
* This class encapsulates all state management for buffering and writing
* through to the container.
*/
public class BlockOutputStream extends OutputStream {
public static final Logger LOG =
LoggerFactory.getLogger(BlockOutputStream.class);
private volatile BlockID blockID;
private final BlockData.Builder containerBlockData;
private XceiverClientManager xceiverClientManager;
private XceiverClientSpi xceiverClient;
private final ContainerProtos.ChecksumType checksumType;
private final int bytesPerChecksum;
private int chunkIndex;
private int chunkSize;
private final long streamBufferFlushSize;
private final long streamBufferMaxSize;
private BufferPool bufferPool;
// The IOException will be set by response handling thread in case there is an
// exception received in the response. If the exception is set, the next
// request will fail upfront.
private AtomicReference<IOException> ioException;
private ExecutorService responseExecutor;
// the effective length of data flushed so far
private long totalDataFlushedLength;
// effective data write attempted so far for the block
private long writtenDataLength;
// List containing buffers for which the putBlock call will
// update the length in the datanodes. This list will just maintain
// references to the buffers in the BufferPool which will be cleared
// when the watchForCommit acknowledges a putBlock logIndex has been
// committed on all datanodes. This list will be a place holder for buffers
// which got written between successive putBlock calls.
private List<ByteBuffer> bufferList;
// This object will maintain the commitIndexes and byteBufferList in order
// Also, corresponding to the logIndex, the corresponding list of buffers will
// be released from the buffer pool.
private final CommitWatcher commitWatcher;
private List<DatanodeDetails> failedServers;
/**
* Creates a new BlockOutputStream.
*
* @param blockID block ID
* @param xceiverClientManager client manager that controls client
* @param pipeline pipeline where block will be written
* @param chunkSize chunk size
* @param bufferPool pool of buffers
* @param streamBufferFlushSize flush size
* @param streamBufferMaxSize max size of the currentBuffer
* @param watchTimeout watch timeout
* @param checksumType checksum type
* @param bytesPerChecksum Bytes per checksum
*/
@SuppressWarnings("parameternumber")
public BlockOutputStream(BlockID blockID,
XceiverClientManager xceiverClientManager, Pipeline pipeline,
int chunkSize, long streamBufferFlushSize, long streamBufferMaxSize,
long watchTimeout, BufferPool bufferPool, ChecksumType checksumType,
int bytesPerChecksum)
throws IOException {
this.blockID = blockID;
this.chunkSize = chunkSize;
KeyValue keyValue =
KeyValue.newBuilder().setKey("TYPE").setValue("KEY").build();
this.containerBlockData =
BlockData.newBuilder().setBlockID(blockID.getDatanodeBlockIDProtobuf())
.addMetadata(keyValue);
this.xceiverClientManager = xceiverClientManager;
this.xceiverClient = xceiverClientManager.acquireClient(pipeline);
this.chunkIndex = 0;
this.streamBufferFlushSize = streamBufferFlushSize;
this.streamBufferMaxSize = streamBufferMaxSize;
this.bufferPool = bufferPool;
this.checksumType = checksumType;
this.bytesPerChecksum = bytesPerChecksum;
// A single thread executor handle the responses of async requests
responseExecutor = Executors.newSingleThreadExecutor();
commitWatcher = new CommitWatcher(bufferPool, xceiverClient, watchTimeout);
bufferList = null;
totalDataFlushedLength = 0;
writtenDataLength = 0;
failedServers = new ArrayList<>(0);
ioException = new AtomicReference<>(null);
}
public BlockID getBlockID() {
return blockID;
}
public long getTotalAckDataLength() {
return commitWatcher.getTotalAckDataLength();
}
public long getWrittenDataLength() {
return writtenDataLength;
}
public List<DatanodeDetails> getFailedServers() {
return failedServers;
}
@VisibleForTesting
public XceiverClientSpi getXceiverClient() {
return xceiverClient;
}
@VisibleForTesting
public long getTotalDataFlushedLength() {
return totalDataFlushedLength;
}
@VisibleForTesting
public BufferPool getBufferPool() {
return bufferPool;
}
public IOException getIoException() {
return ioException.get();
}
@VisibleForTesting
public Map<Long, List<ByteBuffer>> getCommitIndex2flushedDataMap() {
return commitWatcher.getCommitIndex2flushedDataMap();
}
@Override
public void write(int b) throws IOException {
checkOpen();
byte[] buf = new byte[1];
buf[0] = (byte) b;
write(buf, 0, 1);
}
@Override
public void write(byte[] b, int off, int len) throws IOException {
checkOpen();
if (b == null) {
throw new NullPointerException();
}
if ((off < 0) || (off > b.length) || (len < 0) || ((off + len) > b.length)
|| ((off + len) < 0)) {
throw new IndexOutOfBoundsException();
}
if (len == 0) {
return;
}
while (len > 0) {
int writeLen;
// Allocate a buffer if needed. The buffer will be allocated only
// once as needed and will be reused again for multiple blockOutputStream
// entries.
ByteBuffer currentBuffer = bufferPool.allocateBufferIfNeeded();
int pos = currentBuffer.position();
writeLen =
Math.min(chunkSize - pos % chunkSize, len);
currentBuffer.put(b, off, writeLen);
if (!currentBuffer.hasRemaining()) {
writeChunk(currentBuffer);
}
off += writeLen;
len -= writeLen;
writtenDataLength += writeLen;
if (shouldFlush()) {
updateFlushLength();
executePutBlock();
}
// Data in the bufferPool can not exceed streamBufferMaxSize
if (isBufferPoolFull()) {
handleFullBuffer();
}
}
}
private boolean shouldFlush() {
return bufferPool.computeBufferData() % streamBufferFlushSize == 0;
}
private void updateFlushLength() {
totalDataFlushedLength += writtenDataLength - totalDataFlushedLength;
}
private boolean isBufferPoolFull() {
return bufferPool.computeBufferData() == streamBufferMaxSize;
}
/**
* Will be called on the retryPath in case closedContainerException/
* TimeoutException.
* @param len length of data to write
* @throws IOException if error occurred
*/
// In this case, the data is already cached in the currentBuffer.
public void writeOnRetry(long len) throws IOException {
if (len == 0) {
return;
}
int count = 0;
Preconditions.checkArgument(len <= streamBufferMaxSize);
while (len > 0) {
long writeLen;
writeLen = Math.min(chunkSize, len);
if (writeLen == chunkSize) {
writeChunk(bufferPool.getBuffer(count));
}
len -= writeLen;
count++;
writtenDataLength += writeLen;
// we should not call isBufferFull/shouldFlush here.
// The buffer might already be full as whole data is already cached in
// the buffer. We should just validate
// if we wrote data of size streamBufferMaxSize/streamBufferFlushSize to
// call for handling full buffer/flush buffer condition.
if (writtenDataLength % streamBufferFlushSize == 0) {
// reset the position to zero as now we will be reading the
// next buffer in the list
updateFlushLength();
executePutBlock();
}
if (writtenDataLength == streamBufferMaxSize) {
handleFullBuffer();
}
}
}
/**
* This is a blocking call. It will wait for the flush till the commit index
* at the head of the commitIndex2flushedDataMap gets replicated to all or
* majority.
* @throws IOException
*/
private void handleFullBuffer() throws IOException {
try {
checkOpen();
if (!commitWatcher.getFutureMap().isEmpty()) {
waitOnFlushFutures();
}
} catch (InterruptedException | ExecutionException e) {
setIoException(e);
adjustBuffersOnException();
throw getIoException();
}
watchForCommit(true);
}
// It may happen that once the exception is encountered , we still might
// have successfully flushed up to a certain index. Make sure the buffers
// only contain data which have not been sufficiently replicated
private void adjustBuffersOnException() {
commitWatcher.releaseBuffersOnException();
}
/**
* calls watchForCommit API of the Ratis Client. For Standalone client,
* it is a no op.
* @param bufferFull flag indicating whether bufferFull condition is hit or
* its called as part flush/close
* @return minimum commit index replicated to all nodes
* @throws IOException IOException in case watch gets timed out
*/
private void watchForCommit(boolean bufferFull) throws IOException {
checkOpen();
try {
XceiverClientReply reply = bufferFull ?
commitWatcher.watchOnFirstIndex() : commitWatcher.watchOnLastIndex();
if (reply != null) {
List<DatanodeDetails> dnList = reply.getDatanodes();
if (!dnList.isEmpty()) {
Pipeline pipe = xceiverClient.getPipeline();
LOG.warn("Failed to commit BlockId {} on {}. Failed nodes: {}",
blockID, pipe, dnList);
failedServers.addAll(dnList);
}
}
} catch (IOException ioe) {
setIoException(ioe);
throw getIoException();
}
}
private CompletableFuture<ContainerProtos.
ContainerCommandResponseProto> executePutBlock()
throws IOException {
checkOpen();
long flushPos = totalDataFlushedLength;
Preconditions.checkNotNull(bufferList);
List<ByteBuffer> byteBufferList = bufferList;
bufferList = null;
Preconditions.checkNotNull(byteBufferList);
CompletableFuture<ContainerProtos.
ContainerCommandResponseProto> flushFuture;
try {
XceiverClientReply asyncReply =
putBlockAsync(xceiverClient, containerBlockData.build());
CompletableFuture<ContainerProtos.ContainerCommandResponseProto> future =
asyncReply.getResponse();
flushFuture = future.thenApplyAsync(e -> {
try {
validateResponse(e);
} catch (IOException sce) {
throw new CompletionException(sce);
}
// if the ioException is not set, putBlock is successful
if (getIoException() == null) {
BlockID responseBlockID = BlockID.getFromProtobuf(
e.getPutBlock().getCommittedBlockLength().getBlockID());
Preconditions.checkState(blockID.getContainerBlockID()
.equals(responseBlockID.getContainerBlockID()));
// updates the bcsId of the block
blockID = responseBlockID;
if (LOG.isDebugEnabled()) {
LOG.debug(
"Adding index " + asyncReply.getLogIndex() + " commitMap size "
+ commitWatcher.getCommitInfoMapSize() + " flushLength "
+ flushPos + " numBuffers " + byteBufferList.size()
+ " blockID " + blockID + " bufferPool size" + bufferPool
.getSize() + " currentBufferIndex " + bufferPool
.getCurrentBufferIndex());
}
// for standalone protocol, logIndex will always be 0.
commitWatcher
.updateCommitInfoMap(asyncReply.getLogIndex(), byteBufferList);
}
return e;
}, responseExecutor).exceptionally(e -> {
if (LOG.isDebugEnabled()) {
LOG.debug(
"putBlock failed for blockID " + blockID + " with exception " + e
.getLocalizedMessage());
}
CompletionException ce = new CompletionException(e);
setIoException(ce);
throw ce;
});
} catch (IOException | InterruptedException | ExecutionException e) {
throw new IOException(
"Unexpected Storage Container Exception: " + e.toString(), e);
}
commitWatcher.getFutureMap().put(flushPos, flushFuture);
return flushFuture;
}
@Override
public void flush() throws IOException {
if (xceiverClientManager != null && xceiverClient != null
&& bufferPool != null && bufferPool.getSize() > 0) {
try {
handleFlush();
} catch (InterruptedException | ExecutionException e) {
// just set the exception here as well in order to maintain sanctity of
// ioException field
setIoException(e);
adjustBuffersOnException();
throw getIoException();
}
}
}
private void writeChunk(ByteBuffer buffer)
throws IOException {
// This data in the buffer will be pushed to datanode and a reference will
// be added to the bufferList. Once putBlock gets executed, this list will
// be marked null. Hence, during first writeChunk call after every putBlock
// call or during the first call to writeChunk here, the list will be null.
if (bufferList == null) {
bufferList = new ArrayList<>();
}
bufferList.add(buffer);
// Please note : We are not flipping the slice when we write since
// the slices are pointing the currentBuffer start and end as needed for
// the chunk write. Also please note, Duplicate does not create a
// copy of data, it only creates metadata that points to the data
// stream.
ByteBuffer chunk = buffer.duplicate();
chunk.position(0);
chunk.limit(buffer.position());
writeChunkToContainer(chunk);
}
private void handleFlush()
throws IOException, InterruptedException, ExecutionException {
checkOpen();
// flush the last chunk data residing on the currentBuffer
if (totalDataFlushedLength < writtenDataLength) {
ByteBuffer currentBuffer = bufferPool.getCurrentBuffer();
Preconditions.checkArgument(currentBuffer.position() > 0);
if (currentBuffer.position() != chunkSize) {
writeChunk(currentBuffer);
}
// This can be a partially filled chunk. Since we are flushing the buffer
// here, we just limit this buffer to the current position. So that next
// write will happen in new buffer
updateFlushLength();
executePutBlock();
}
waitOnFlushFutures();
watchForCommit(false);
// just check again if the exception is hit while waiting for the
// futures to ensure flush has indeed succeeded
// irrespective of whether the commitIndex2flushedDataMap is empty
// or not, ensure there is no exception set
checkOpen();
}
@Override
public void close() throws IOException {
if (xceiverClientManager != null && xceiverClient != null
&& bufferPool != null && bufferPool.getSize() > 0) {
try {
handleFlush();
} catch (InterruptedException | ExecutionException e) {
setIoException(e);
adjustBuffersOnException();
throw getIoException();
} finally {
cleanup(false);
}
// TODO: Turn the below buffer empty check on when Standalone pipeline
// is removed in the write path in tests
// Preconditions.checkArgument(buffer.position() == 0);
// bufferPool.checkBufferPoolEmpty();
}
}
private void waitOnFlushFutures()
throws InterruptedException, ExecutionException {
CompletableFuture<Void> combinedFuture = CompletableFuture.allOf(
commitWatcher.getFutureMap().values().toArray(
new CompletableFuture[commitWatcher.getFutureMap().size()]));
// wait for all the transactions to complete
combinedFuture.get();
}
private void validateResponse(
ContainerProtos.ContainerCommandResponseProto responseProto)
throws IOException {
try {
// if the ioException is already set, it means a prev request has failed
// just throw the exception. The current operation will fail with the
// original error
IOException exception = getIoException();
if (exception != null) {
throw exception;
}
ContainerProtocolCalls.validateContainerResponse(responseProto);
} catch (StorageContainerException sce) {
LOG.error("Unexpected Storage Container Exception: ", sce);
setIoException(sce);
throw sce;
}
}
private void setIoException(Exception e) {
if (getIoException() == null) {
IOException exception = new IOException(
"Unexpected Storage Container Exception: " + e.toString(), e);
ioException.compareAndSet(null, exception);
}
}
public void cleanup(boolean invalidateClient) {
if (xceiverClientManager != null) {
xceiverClientManager.releaseClient(xceiverClient, invalidateClient);
}
xceiverClientManager = null;
xceiverClient = null;
commitWatcher.cleanup();
if (bufferList != null) {
bufferList.clear();
}
bufferList = null;
responseExecutor.shutdown();
}
/**
* Checks if the stream is open or exception has occured.
* If not, throws an exception.
*
* @throws IOException if stream is closed
*/
private void checkOpen() throws IOException {
if (isClosed()) {
throw new IOException("BlockOutputStream has been closed.");
} else if (getIoException() != null) {
adjustBuffersOnException();
throw getIoException();
}
}
public boolean isClosed() {
return xceiverClient == null;
}
/**
* Writes buffered data as a new chunk to the container and saves chunk
* information to be used later in putKey call.
*
* @throws IOException if there is an I/O error while performing the call
* @throws OzoneChecksumException if there is an error while computing
* checksum
*/
private void writeChunkToContainer(ByteBuffer chunk) throws IOException {
int effectiveChunkSize = chunk.remaining();
ByteString data = bufferPool.byteStringConversion().apply(chunk);
Checksum checksum = new Checksum(checksumType, bytesPerChecksum);
ChecksumData checksumData = checksum.computeChecksum(chunk);
ChunkInfo chunkInfo = ChunkInfo.newBuilder()
.setChunkName(blockID.getLocalID() + "_chunk_" + ++chunkIndex)
.setOffset(0)
.setLen(effectiveChunkSize)
.setChecksumData(checksumData.getProtoBufMessage())
.build();
try {
XceiverClientReply asyncReply =
writeChunkAsync(xceiverClient, chunkInfo, blockID, data);
CompletableFuture<ContainerProtos.ContainerCommandResponseProto> future =
asyncReply.getResponse();
future.thenApplyAsync(e -> {
try {
validateResponse(e);
} catch (IOException sce) {
future.completeExceptionally(sce);
}
return e;
}, responseExecutor).exceptionally(e -> {
if (LOG.isDebugEnabled()) {
LOG.debug(
"writing chunk failed " + chunkInfo.getChunkName() + " blockID "
+ blockID + " with exception " + e.getLocalizedMessage());
}
CompletionException ce = new CompletionException(e);
setIoException(ce);
throw ce;
});
} catch (IOException | InterruptedException | ExecutionException e) {
throw new IOException(
"Unexpected Storage Container Exception: " + e.toString(), e);
}
if (LOG.isDebugEnabled()) {
LOG.debug(
"writing chunk " + chunkInfo.getChunkName() + " blockID " + blockID
+ " length " + effectiveChunkSize);
}
containerBlockData.addChunks(chunkInfo);
}
@VisibleForTesting
public void setXceiverClient(XceiverClientSpi xceiverClient) {
this.xceiverClient = xceiverClient;
}
}

129
hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BufferPool.java
View File

@ -1,129 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdds.scm.storage;
import com.google.common.base.Preconditions;
import org.apache.hadoop.hdds.scm.ByteStringConversion;
import org.apache.ratis.thirdparty.com.google.protobuf.ByteString;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.List;
import java.util.function.Function;
/**
* This class creates and manages pool of n buffers.
*/
public class BufferPool {
private List<ByteBuffer> bufferList;
private int currentBufferIndex;
private final int bufferSize;
private final int capacity;
private final Function<ByteBuffer, ByteString> byteStringConversion;
public BufferPool(int bufferSize, int capacity) {
this(bufferSize, capacity,
ByteStringConversion.createByteBufferConversion(null));
}
public BufferPool(int bufferSize, int capacity,
Function<ByteBuffer, ByteString> byteStringConversion){
this.capacity = capacity;
this.bufferSize = bufferSize;
bufferList = new ArrayList<>(capacity);
currentBufferIndex = -1;
this.byteStringConversion = byteStringConversion;
}
public Function<ByteBuffer, ByteString> byteStringConversion(){
return byteStringConversion;
}
public ByteBuffer getCurrentBuffer() {
return currentBufferIndex == -1 ? null : bufferList.get(currentBufferIndex);
}
/**
* If the currentBufferIndex is less than the buffer size - 1,
* it means, the next buffer in the list has been freed up for
* rewriting. Reuse the next available buffer in such cases.
*
* In case, the currentBufferIndex == buffer.size and buffer size is still
* less than the capacity to be allocated, just allocate a buffer of size
* chunk size.
*
*/
public ByteBuffer allocateBufferIfNeeded() {
ByteBuffer buffer = getCurrentBuffer();
if (buffer != null && buffer.hasRemaining()) {
return buffer;
}
if (currentBufferIndex < bufferList.size() - 1) {
buffer = getBuffer(currentBufferIndex + 1);
} else {
buffer = ByteBuffer.allocate(bufferSize);
bufferList.add(buffer);
}
Preconditions.checkArgument(bufferList.size() <= capacity);
currentBufferIndex++;
// TODO: Turn the below precondition check on when Standalone pipeline
// is removed in the write path in tests
// Preconditions.checkArgument(buffer.position() == 0);
return buffer;
}
public void releaseBuffer(ByteBuffer byteBuffer) {
// always remove from head of the list and append at last
ByteBuffer buffer = bufferList.remove(0);
// Ensure the buffer to be removed is always at the head of the list.
Preconditions.checkArgument(buffer.equals(byteBuffer));
buffer.clear();
bufferList.add(buffer);
Preconditions.checkArgument(currentBufferIndex >= 0);
currentBufferIndex--;
}
public void clearBufferPool() {
bufferList.clear();
currentBufferIndex = -1;
}
public void checkBufferPoolEmpty() {
Preconditions.checkArgument(computeBufferData() == 0);
}
public long computeBufferData() {
return bufferList.stream().mapToInt(value -> value.position())
.sum();
}
public int getSize() {
return bufferList.size();
}
public ByteBuffer getBuffer(int index) {
return bufferList.get(index);
}
int getCurrentBufferIndex() {
return currentBufferIndex;
}
}

544
hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/ChunkInputStream.java
View File

@ -1,544 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdds.scm.storage;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import org.apache.hadoop.fs.Seekable;
import org.apache.hadoop.hdds.client.BlockID;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandRequestProto;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandResponseProto;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ChunkInfo;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ReadChunkResponseProto;
import org.apache.hadoop.hdds.scm.XceiverClientSpi;
import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException;
import org.apache.hadoop.ozone.common.Checksum;
import org.apache.hadoop.ozone.common.ChecksumData;
import org.apache.hadoop.ozone.common.OzoneChecksumException;
import org.apache.ratis.thirdparty.com.google.protobuf.ByteString;
import java.io.EOFException;
import java.io.IOException;
import java.io.InputStream;
import java.nio.ByteBuffer;
import java.util.List;
/**
* An {@link InputStream} called from BlockInputStream to read a chunk from the
* container. Each chunk may contain multiple underlying {@link ByteBuffer}
* instances.
*/
public class ChunkInputStream extends InputStream implements Seekable {
private ChunkInfo chunkInfo;
private final long length;
private final BlockID blockID;
private XceiverClientSpi xceiverClient;
private boolean verifyChecksum;
private boolean allocated = false;
// Buffer to store the chunk data read from the DN container
private List<ByteBuffer> buffers;
// Index of the buffers corresponding to the current position of the buffers
private int bufferIndex;
// The offset of the current data residing in the buffers w.r.t the start
// of chunk data
private long bufferOffset;
// The number of bytes of chunk data residing in the buffers currently
private long bufferLength;
// Position of the ChunkInputStream is maintained by this variable (if a
// seek is performed. This position is w.r.t to the chunk only and not the
// block or key. This variable is set only if either the buffers are not
// yet allocated or the if the allocated buffers do not cover the seeked
// position. Once the chunk is read, this variable is reset.
private long chunkPosition = -1;
private static final int EOF = -1;
ChunkInputStream(ChunkInfo chunkInfo, BlockID blockId,
XceiverClientSpi xceiverClient, boolean verifyChecksum) {
this.chunkInfo = chunkInfo;
this.length = chunkInfo.getLen();
this.blockID = blockId;
this.xceiverClient = xceiverClient;
this.verifyChecksum = verifyChecksum;
}
public synchronized long getRemaining() throws IOException {
return length - getPos();
}
/**
* {@inheritDoc}
*/
@Override
public synchronized int read() throws IOException {
checkOpen();
int available = prepareRead(1);
int dataout = EOF;
if (available == EOF) {
// There is no more data in the chunk stream. The buffers should have
// been released by now
Preconditions.checkState(buffers == null);
} else {
dataout = Byte.toUnsignedInt(buffers.get(bufferIndex).get());
}
if (chunkStreamEOF()) {
// consumer might use getPos to determine EOF,
// so release buffers when serving the last byte of data
releaseBuffers();
}
return dataout;
}
/**
* {@inheritDoc}
*/
@Override
public synchronized int read(byte[] b, int off, int len) throws IOException {
// According to the JavaDocs for InputStream, it is recommended that
// subclasses provide an override of bulk read if possible for performance
// reasons. In addition to performance, we need to do it for correctness
// reasons. The Ozone REST service uses PipedInputStream and
// PipedOutputStream to relay HTTP response data between a Jersey thread and
// a Netty thread. It turns out that PipedInputStream/PipedOutputStream
// have a subtle dependency (bug?) on the wrapped stream providing separate
// implementations of single-byte read and bulk read. Without this, get key
// responses might close the connection before writing all of the bytes
// advertised in the Content-Length.
if (b == null) {
throw new NullPointerException();
}
if (off < 0 || len < 0 || len > b.length - off) {
throw new IndexOutOfBoundsException();
}
if (len == 0) {
return 0;
}
checkOpen();
int total = 0;
while (len > 0) {
int available = prepareRead(len);
if (available == EOF) {
// There is no more data in the chunk stream. The buffers should have
// been released by now
Preconditions.checkState(buffers == null);
return total != 0 ? total : EOF;
}
buffers.get(bufferIndex).get(b, off + total, available);
len -= available;
total += available;
}
if (chunkStreamEOF()) {
// smart consumers determine EOF by calling getPos()
// so we release buffers when serving the final bytes of data
releaseBuffers();
}
return total;
}
/**
* Seeks the ChunkInputStream to the specified position. This is done by
* updating the chunkPosition to the seeked position in case the buffers
* are not allocated or buffers do not contain the data corresponding to
* the seeked position (determined by buffersHavePosition()). Otherwise,
* the buffers position is updated to the seeked position.
*/
@Override
public synchronized void seek(long pos) throws IOException {
if (pos < 0 || pos >= length) {
if (pos == 0) {
// It is possible for length and pos to be zero in which case
// seek should return instead of throwing exception
return;
}
throw new EOFException("EOF encountered at pos: " + pos + " for chunk: "
+ chunkInfo.getChunkName());
}
if (buffersHavePosition(pos)) {
// The bufferPosition is w.r.t the current chunk.
// Adjust the bufferIndex and position to the seeked position.
adjustBufferPosition(pos - bufferOffset);
} else {
chunkPosition = pos;
}
}
@Override
public synchronized long getPos() throws IOException {
if (chunkPosition >= 0) {
return chunkPosition;
}
if (chunkStreamEOF()) {
return length;
}
if (buffersHaveData()) {
return bufferOffset + buffers.get(bufferIndex).position();
}
if (buffersAllocated()) {
return bufferOffset + bufferLength;
}
return 0;
}
@Override
public boolean seekToNewSource(long targetPos) throws IOException {
return false;
}
@Override
public synchronized void close() {
if (xceiverClient != null) {
xceiverClient = null;
}
}
/**
* Checks if the stream is open. If not, throw an exception.
*
* @throws IOException if stream is closed
*/
protected synchronized void checkOpen() throws IOException {
if (xceiverClient == null) {
throw new IOException("BlockInputStream has been closed.");
}
}
/**
* Prepares to read by advancing through buffers or allocating new buffers,
* as needed until it finds data to return, or encounters EOF.
* @param len desired lenght of data to read
* @return length of data available to read, possibly less than desired length
*/
private synchronized int prepareRead(int len) throws IOException {
for (;;) {
if (chunkPosition >= 0) {
if (buffersHavePosition(chunkPosition)) {
// The current buffers have the seeked position. Adjust the buffer
// index and position to point to the chunkPosition.
adjustBufferPosition(chunkPosition - bufferOffset);
} else {
// Read a required chunk data to fill the buffers with seeked
// position data
readChunkFromContainer(len);
}
}
if (buffersHaveData()) {
// Data is available from buffers
ByteBuffer bb = buffers.get(bufferIndex);
return len > bb.remaining() ? bb.remaining() : len;
} else if (dataRemainingInChunk()) {
// There is more data in the chunk stream which has not
// been read into the buffers yet.
readChunkFromContainer(len);
} else {
// All available input from this chunk stream has been consumed.
return EOF;
}
}
}
/**
* Reads full or partial Chunk from DN Container based on the current
* position of the ChunkInputStream, the number of bytes of data to read
* and the checksum boundaries.
* If successful, then the read data in saved in the buffers so that
* subsequent read calls can utilize it.
* @param len number of bytes of data to be read
* @throws IOException if there is an I/O error while performing the call
* to Datanode
*/
private synchronized void readChunkFromContainer(int len) throws IOException {
// index of first byte to be read from the chunk
long startByteIndex;
if (chunkPosition >= 0) {
// If seek operation was called to advance the buffer position, the
// chunk should be read from that position onwards.
startByteIndex = chunkPosition;
} else {
// Start reading the chunk from the last chunkPosition onwards.
startByteIndex = bufferOffset + bufferLength;
}
if (verifyChecksum) {
// Update the bufferOffset and bufferLength as per the checksum
// boundary requirement.
computeChecksumBoundaries(startByteIndex, len);
} else {
// Read from the startByteIndex
bufferOffset = startByteIndex;
bufferLength = len;
}
// Adjust the chunkInfo so that only the required bytes are read from
// the chunk.
final ChunkInfo adjustedChunkInfo = ChunkInfo.newBuilder(chunkInfo)
.setOffset(bufferOffset)
.setLen(bufferLength)
.build();
ByteString byteString = readChunk(adjustedChunkInfo);
buffers = byteString.asReadOnlyByteBufferList();
bufferIndex = 0;
allocated = true;
// If the stream was seeked to position before, then the buffer
// position should be adjusted as the reads happen at checksum boundaries.
// The buffers position might need to be adjusted for the following
// scenarios:
// 1. Stream was seeked to a position before the chunk was read
// 2. Chunk was read from index < the current position to account for
// checksum boundaries.
adjustBufferPosition(startByteIndex - bufferOffset);
}
/**
* Send RPC call to get the chunk from the container.
*/
@VisibleForTesting
protected ByteString readChunk(ChunkInfo readChunkInfo) throws IOException {
ReadChunkResponseProto readChunkResponse;
try {
List<CheckedBiFunction> validators =
ContainerProtocolCalls.getValidatorList();
validators.add(validator);
readChunkResponse = ContainerProtocolCalls.readChunk(xceiverClient,
readChunkInfo, blockID, validators);
} catch (IOException e) {
if (e instanceof StorageContainerException) {
throw e;
}
throw new IOException("Unexpected OzoneException: " + e.toString(), e);
}
return readChunkResponse.getData();
}
private CheckedBiFunction<ContainerCommandRequestProto,
ContainerCommandResponseProto, IOException> validator =
(request, response) -> {
final ChunkInfo reqChunkInfo =
request.getReadChunk().getChunkData();
ReadChunkResponseProto readChunkResponse = response.getReadChunk();
ByteString byteString = readChunkResponse.getData();
if (byteString.size() != reqChunkInfo.getLen()) {
// Bytes read from chunk should be equal to chunk size.
throw new OzoneChecksumException(String
.format("Inconsistent read for chunk=%s len=%d bytesRead=%d",
reqChunkInfo.getChunkName(), reqChunkInfo.getLen(),
byteString.size()));
}
if (verifyChecksum) {
ChecksumData checksumData = ChecksumData.getFromProtoBuf(
chunkInfo.getChecksumData());
// ChecksumData stores checksum for each 'numBytesPerChecksum'
// number of bytes in a list. Compute the index of the first
// checksum to match with the read data
int checkumStartIndex = (int) (reqChunkInfo.getOffset() /
checksumData.getBytesPerChecksum());
Checksum.verifyChecksum(
byteString, checksumData, checkumStartIndex);
}
};
/**
* Return the offset and length of bytes that need to be read from the
* chunk file to cover the checksum boundaries covering the actual start and
* end of the chunk index to be read.
* For example, lets say the client is reading from index 120 to 450 in the
* chunk. And let's say checksum is stored for every 100 bytes in the chunk
* i.e. the first checksum is for bytes from index 0 to 99, the next for
* bytes from index 100 to 199 and so on. To verify bytes from 120 to 450,
* we would need to read from bytes 100 to 499 so that checksum
* verification can be done.
*
* @param startByteIndex the first byte index to be read by client
* @param dataLen number of bytes to be read from the chunk
*/
private void computeChecksumBoundaries(long startByteIndex, int dataLen) {
int bytesPerChecksum = chunkInfo.getChecksumData().getBytesPerChecksum();
// index of the last byte to be read from chunk, inclusively.
final long endByteIndex = startByteIndex + dataLen - 1;
bufferOffset = (startByteIndex / bytesPerChecksum)
* bytesPerChecksum; // inclusive
final long endIndex = ((endByteIndex / bytesPerChecksum) + 1)
* bytesPerChecksum; // exclusive
bufferLength = Math.min(endIndex, length) - bufferOffset;
}
/**
* Adjust the buffers position to account for seeked position and/ or checksum
* boundary reads.
* @param bufferPosition the position to which the buffers must be advanced
*/
private void adjustBufferPosition(long bufferPosition) {
// The bufferPosition is w.r.t the current chunk.
// Adjust the bufferIndex and position to the seeked chunkPosition.
long tempOffest = 0;
for (int i = 0; i < buffers.size(); i++) {
if (bufferPosition - tempOffest >= buffers.get(i).capacity()) {
tempOffest += buffers.get(i).capacity();
} else {
bufferIndex = i;
break;
}
}
buffers.get(bufferIndex).position((int) (bufferPosition - tempOffest));
// Reset the chunkPosition as chunk stream has been initialized i.e. the
// buffers have been allocated.
resetPosition();
}
/**
* Check if the buffers have been allocated data and false otherwise.
*/
private boolean buffersAllocated() {
return buffers != null && !buffers.isEmpty();
}
/**
* Check if the buffers have any data remaining between the current
* position and the limit.
*/
private boolean buffersHaveData() {
boolean hasData = false;
if (buffersAllocated()) {
while (bufferIndex < (buffers.size())) {
if (buffers.get(bufferIndex).hasRemaining()) {
// current buffer has data
hasData = true;
break;
} else {
if (buffersRemaining()) {
// move to next available buffer
++bufferIndex;
Preconditions.checkState(bufferIndex < buffers.size());
} else {
// no more buffers remaining
break;
}
}
}
}
return hasData;
}
private boolean buffersRemaining() {
return (bufferIndex < (buffers.size() - 1));
}
/**
* Check if curernt buffers have the data corresponding to the input position.
*/
private boolean buffersHavePosition(long pos) {
// Check if buffers have been allocated
if (buffersAllocated()) {
// Check if the current buffers cover the input position
return pos >= bufferOffset &&
pos < bufferOffset + bufferLength;
}
return false;
}
/**
* Check if there is more data in the chunk which has not yet been read
* into the buffers.
*/
private boolean dataRemainingInChunk() {
long bufferPos;
if (chunkPosition >= 0) {
bufferPos = chunkPosition;
} else {
bufferPos = bufferOffset + bufferLength;
}
return bufferPos < length;
}
/**
* Check if end of chunkStream has been reached.
*/
private boolean chunkStreamEOF() {
if (!allocated) {
// Chunk data has not been read yet
return false;
}
if (buffersHaveData() || dataRemainingInChunk()) {
return false;
} else {
Preconditions.checkState(bufferOffset + bufferLength == length,
"EOF detected, but not at the last byte of the chunk");
return true;
}
}
/**
* If EOF is reached, release the buffers.
*/
private void releaseBuffers() {
buffers = null;
bufferIndex = 0;
}
/**
* Reset the chunkPosition once the buffers are allocated.
*/
void resetPosition() {
this.chunkPosition = -1;
}
String getChunkName() {
return chunkInfo.getChunkName();
}
protected long getLength() {
return length;
}
@VisibleForTesting
protected long getChunkPosition() {
return chunkPosition;
}
}

240
hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/CommitWatcher.java
View File

@ -1,240 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* This class maintains the map of the commitIndexes to be watched for
* successful replication in the datanodes in a given pipeline. It also releases
* the buffers associated with the user data back to {@Link BufferPool} once
* minimum replication criteria is achieved during an ozone key write.
*/
package org.apache.hadoop.hdds.scm.storage;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
import org.apache.hadoop.hdds.scm.XceiverClientReply;
import org.apache.hadoop.hdds.scm.XceiverClientSpi;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.List;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ConcurrentSkipListMap;
import java.util.stream.Collectors;
/**
* This class executes watchForCommit on ratis pipeline and releases
* buffers once data successfully gets replicated.
*/
public class CommitWatcher {
private static final Logger LOG =
LoggerFactory.getLogger(CommitWatcher.class);
// A reference to the pool of buffers holding the data
private BufferPool bufferPool;
// The map should maintain the keys (logIndexes) in order so that while
// removing we always end up updating incremented data flushed length.
// Also, corresponding to the logIndex, the corresponding list of buffers will
// be released from the buffer pool.
private ConcurrentSkipListMap<Long, List<ByteBuffer>>
commitIndex2flushedDataMap;
// future Map to hold up all putBlock futures
private ConcurrentHashMap<Long,
CompletableFuture<ContainerProtos.ContainerCommandResponseProto>>
futureMap;
private XceiverClientSpi xceiverClient;
private final long watchTimeout;
// total data which has been successfully flushed and acknowledged
// by all servers
private long totalAckDataLength;
public CommitWatcher(BufferPool bufferPool, XceiverClientSpi xceiverClient,
long watchTimeout) {
this.bufferPool = bufferPool;
this.xceiverClient = xceiverClient;
this.watchTimeout = watchTimeout;
commitIndex2flushedDataMap = new ConcurrentSkipListMap<>();
totalAckDataLength = 0;
futureMap = new ConcurrentHashMap<>();
}
/**
* just update the totalAckDataLength. In case of failure,
* we will read the data starting from totalAckDataLength.
*/
private long releaseBuffers(List<Long> indexes) {
Preconditions.checkArgument(!commitIndex2flushedDataMap.isEmpty());
for (long index : indexes) {
Preconditions.checkState(commitIndex2flushedDataMap.containsKey(index));
List<ByteBuffer> buffers = commitIndex2flushedDataMap.remove(index);
long length = buffers.stream().mapToLong(value -> {
int pos = value.position();
return pos;
}).sum();
totalAckDataLength += length;
// clear the future object from the future Map
Preconditions.checkNotNull(futureMap.remove(totalAckDataLength));
for (ByteBuffer byteBuffer : buffers) {
bufferPool.releaseBuffer(byteBuffer);
}
}
return totalAckDataLength;
}
public void updateCommitInfoMap(long index, List<ByteBuffer> byteBufferList) {
commitIndex2flushedDataMap
.put(index, byteBufferList);
}
int getCommitInfoMapSize() {
return commitIndex2flushedDataMap.size();
}
/**
* Calls watch for commit for the first index in commitIndex2flushedDataMap to
* the Ratis client.
* @return reply reply from raft client
* @throws IOException in case watchForCommit fails
*/
public XceiverClientReply watchOnFirstIndex() throws IOException {
if (!commitIndex2flushedDataMap.isEmpty()) {
// wait for the first commit index in the commitIndex2flushedDataMap
// to get committed to all or majority of nodes in case timeout
// happens.
long index =
commitIndex2flushedDataMap.keySet().stream().mapToLong(v -> v).min()
.getAsLong();
if (LOG.isDebugEnabled()) {
LOG.debug("waiting for first index " + index + " to catch up");
}
return watchForCommit(index);
} else {
return null;
}
}
/**
* Calls watch for commit for the first index in commitIndex2flushedDataMap to
* the Ratis client.
* @return reply reply from raft client
* @throws IOException in case watchForCommit fails
*/
public XceiverClientReply watchOnLastIndex()
throws IOException {
if (!commitIndex2flushedDataMap.isEmpty()) {
// wait for the commit index in the commitIndex2flushedDataMap
// to get committed to all or majority of nodes in case timeout
// happens.
long index =
commitIndex2flushedDataMap.keySet().stream().mapToLong(v -> v).max()
.getAsLong();
if (LOG.isDebugEnabled()) {
LOG.debug("waiting for last flush Index " + index + " to catch up");
}
return watchForCommit(index);
} else {
return null;
}
}
private void adjustBuffers(long commitIndex) {
List<Long> keyList = commitIndex2flushedDataMap.keySet().stream()
.filter(p -> p <= commitIndex).collect(Collectors.toList());
if (keyList.isEmpty()) {
return;
} else {
releaseBuffers(keyList);
}
}
// It may happen that once the exception is encountered , we still might
// have successfully flushed up to a certain index. Make sure the buffers
// only contain data which have not been sufficiently replicated
void releaseBuffersOnException() {
adjustBuffers(xceiverClient.getReplicatedMinCommitIndex());
}
/**
* calls watchForCommit API of the Ratis Client. For Standalone client,
* it is a no op.
* @param commitIndex log index to watch for
* @return minimum commit index replicated to all nodes
* @throws IOException IOException in case watch gets timed out
*/
public XceiverClientReply watchForCommit(long commitIndex)
throws IOException {
long index;
try {
XceiverClientReply reply =
xceiverClient.watchForCommit(commitIndex, watchTimeout);
if (reply == null) {
index = 0;
} else {
index = reply.getLogIndex();
}
adjustBuffers(index);
return reply;
} catch (TimeoutException | InterruptedException | ExecutionException e) {
LOG.warn("watchForCommit failed for index " + commitIndex, e);
IOException ioException = new IOException(
"Unexpected Storage Container Exception: " + e.toString(), e);
releaseBuffersOnException();
throw ioException;
}
}
@VisibleForTesting
public ConcurrentSkipListMap<Long,
List<ByteBuffer>> getCommitIndex2flushedDataMap() {
return commitIndex2flushedDataMap;
}
public ConcurrentHashMap<Long,
CompletableFuture<ContainerProtos.
ContainerCommandResponseProto>> getFutureMap() {
return futureMap;
}
public long getTotalAckDataLength() {
return totalAckDataLength;
}
public void cleanup() {
if (commitIndex2flushedDataMap != null) {
commitIndex2flushedDataMap.clear();
}
if (futureMap != null) {
futureMap.clear();
}
commitIndex2flushedDataMap = null;
}
}

23
hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/package-info.java
View File

@ -1,23 +0,0 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdds.scm.storage;
/**
* Low level IO streams to upload/download chunks from container service.
*/

234
hadoop-hdds/client/src/test/java/org/apache/hadoop/hdds/scm/storage/TestBlockInputStream.java
View File

@ -1,234 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdds.scm.storage;
import com.google.common.primitives.Bytes;
import org.apache.hadoop.hdds.client.BlockID;
import org.apache.hadoop.hdds.client.ContainerBlockID;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ChecksumType;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ChunkInfo;
import org.apache.hadoop.hdds.scm.XceiverClientManager;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
import org.apache.hadoop.hdds.security.token.OzoneBlockTokenIdentifier;
import org.apache.hadoop.ozone.common.Checksum;
import org.apache.hadoop.security.token.Token;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import java.io.EOFException;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Random;
import static org.apache.hadoop.hdds.scm.storage.TestChunkInputStream.generateRandomData;
/**
* Tests for {@link BlockInputStream}'s functionality.
*/
public class TestBlockInputStream {
private static final int CHUNK_SIZE = 100;
private static Checksum checksum;
private BlockInputStream blockStream;
private byte[] blockData;
private int blockSize;
private List<ChunkInfo> chunks;
private Map<String, byte[]> chunkDataMap;
@Before
public void setup() throws Exception {
BlockID blockID = new BlockID(new ContainerBlockID(1, 1));
checksum = new Checksum(ChecksumType.NONE, CHUNK_SIZE);
createChunkList(5);
blockStream = new DummyBlockInputStream(blockID, blockSize, null, null,
false, null);
}
/**
* Create a mock list of chunks. The first n-1 chunks of length CHUNK_SIZE
* and the last chunk with length CHUNK_SIZE/2.
*/
private void createChunkList(int numChunks)
throws Exception {
chunks = new ArrayList<>(numChunks);
chunkDataMap = new HashMap<>();
blockData = new byte[0];
int i, chunkLen;
byte[] byteData;
String chunkName;
for (i = 0; i < numChunks; i++) {
chunkName = "chunk-" + i;
chunkLen = CHUNK_SIZE;
if (i == numChunks - 1) {
chunkLen = CHUNK_SIZE / 2;
}
byteData = generateRandomData(chunkLen);
ChunkInfo chunkInfo = ChunkInfo.newBuilder()
.setChunkName(chunkName)
.setOffset(0)
.setLen(chunkLen)
.setChecksumData(checksum.computeChecksum(
byteData, 0, chunkLen).getProtoBufMessage())
.build();
chunkDataMap.put(chunkName, byteData);
chunks.add(chunkInfo);
blockSize += chunkLen;
blockData = Bytes.concat(blockData, byteData);
}
}
/**
* A dummy BlockInputStream to mock read block call to DN.
*/
private class DummyBlockInputStream extends BlockInputStream {
DummyBlockInputStream(BlockID blockId,
long blockLen,
Pipeline pipeline,
Token<OzoneBlockTokenIdentifier> token,
boolean verifyChecksum,
XceiverClientManager xceiverClientManager) {
super(blockId, blockLen, pipeline, token, verifyChecksum,
xceiverClientManager);
}
@Override
protected List<ChunkInfo> getChunkInfos() {
return chunks;
}
@Override
protected void addStream(ChunkInfo chunkInfo) {
TestChunkInputStream testChunkInputStream = new TestChunkInputStream();
getChunkStreams().add(testChunkInputStream.new DummyChunkInputStream(
chunkInfo, null, null, false,
chunkDataMap.get(chunkInfo.getChunkName()).clone()));
}
@Override
protected synchronized void checkOpen() throws IOException {
// No action needed
}
}
private void seekAndVerify(int pos) throws Exception {
blockStream.seek(pos);
Assert.assertEquals("Current position of buffer does not match with the " +
"seeked position", pos, blockStream.getPos());
}
/**
* Match readData with the chunkData byte-wise.
* @param readData Data read through ChunkInputStream
* @param inputDataStartIndex first index (inclusive) in chunkData to compare
* with read data
* @param length the number of bytes of data to match starting from
* inputDataStartIndex
*/
private void matchWithInputData(byte[] readData, int inputDataStartIndex,
int length) {
for (int i = inputDataStartIndex; i < inputDataStartIndex + length; i++) {
Assert.assertEquals(blockData[i], readData[i - inputDataStartIndex]);
}
}
@Test
public void testSeek() throws Exception {
// Seek to position 0
int pos = 0;
seekAndVerify(pos);
Assert.assertEquals("ChunkIndex is incorrect", 0,
blockStream.getChunkIndex());
// Before BlockInputStream is initialized (initialization happens during
// read operation), seek should update the BlockInputStream#blockPosition
pos = CHUNK_SIZE;
seekAndVerify(pos);
Assert.assertEquals("ChunkIndex is incorrect", 0,
blockStream.getChunkIndex());
Assert.assertEquals(pos, blockStream.getBlockPosition());
// Initialize the BlockInputStream. After initializtion, the chunkIndex
// should be updated to correspond to the seeked position.
blockStream.initialize();
Assert.assertEquals("ChunkIndex is incorrect", 1,
blockStream.getChunkIndex());
pos = (CHUNK_SIZE * 4) + 5;
seekAndVerify(pos);
Assert.assertEquals("ChunkIndex is incorrect", 4,
blockStream.getChunkIndex());
try {
// Try seeking beyond the blockSize.
pos = blockSize + 10;
seekAndVerify(pos);
Assert.fail("Seek to position beyond block size should fail.");
} catch (EOFException e) {
System.out.println(e);
}
// Seek to random positions between 0 and the block size.
Random random = new Random();
for (int i = 0; i < 10; i++) {
pos = random.nextInt(blockSize);
seekAndVerify(pos);
}
}
@Test
public void testRead() throws Exception {
// read 200 bytes of data starting from position 50. Chunk0 contains
// indices 0 to 99, chunk1 from 100 to 199 and chunk3 from 200 to 299. So
// the read should result in 3 ChunkInputStream reads
seekAndVerify(50);
byte[] b = new byte[200];
blockStream.read(b, 0, 200);
matchWithInputData(b, 50, 200);
// The new position of the blockInputStream should be the last index read
// + 1.
Assert.assertEquals(250, blockStream.getPos());
Assert.assertEquals(2, blockStream.getChunkIndex());
}
@Test
public void testSeekAndRead() throws Exception {
// Seek to a position and read data
seekAndVerify(50);
byte[] b1 = new byte[100];
blockStream.read(b1, 0, 100);
matchWithInputData(b1, 50, 100);
// Next read should start from the position of the last read + 1 i.e. 100
byte[] b2 = new byte[100];
blockStream.read(b2, 0, 100);
matchWithInputData(b2, 150, 100);
}
}

222
hadoop-hdds/client/src/test/java/org/apache/hadoop/hdds/scm/storage/TestChunkInputStream.java
View File

@ -1,222 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdds.scm.storage;
import org.apache.hadoop.hdds.client.BlockID;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ChecksumType;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ChunkInfo;
import org.apache.hadoop.hdds.scm.XceiverClientSpi;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.common.Checksum;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.ratis.thirdparty.com.google.protobuf.ByteString;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import java.io.EOFException;
import java.util.ArrayList;
import java.util.List;
import java.util.Random;
/**
* Tests for {@link ChunkInputStream}'s functionality.
*/
public class TestChunkInputStream {
private static final int CHUNK_SIZE = 100;
private static final int BYTES_PER_CHECKSUM = 20;
private static final String CHUNK_NAME = "dummyChunk";
private static final Random RANDOM = new Random();
private static Checksum checksum;
private DummyChunkInputStream chunkStream;
private ChunkInfo chunkInfo;
private byte[] chunkData;
@Before
public void setup() throws Exception {
checksum = new Checksum(ChecksumType.valueOf(
OzoneConfigKeys.OZONE_CLIENT_CHECKSUM_TYPE_DEFAULT),
BYTES_PER_CHECKSUM);
chunkData = generateRandomData(CHUNK_SIZE);
chunkInfo = ChunkInfo.newBuilder()
.setChunkName(CHUNK_NAME)
.setOffset(0)
.setLen(CHUNK_SIZE)
.setChecksumData(checksum.computeChecksum(
chunkData, 0, CHUNK_SIZE).getProtoBufMessage())
.build();
chunkStream = new DummyChunkInputStream(chunkInfo, null, null, true);
}
static byte[] generateRandomData(int length) {
byte[] bytes = new byte[length];
RANDOM.nextBytes(bytes);
return bytes;
}
/**
* A dummy ChunkInputStream to mock read chunk calls to DN.
*/
public class DummyChunkInputStream extends ChunkInputStream {
// Stores the read chunk data in each readChunk call
private List<ByteString> readByteBuffers = new ArrayList<>();
DummyChunkInputStream(ChunkInfo chunkInfo,
BlockID blockId,
XceiverClientSpi xceiverClient,
boolean verifyChecksum) {
super(chunkInfo, blockId, xceiverClient, verifyChecksum);
}
public DummyChunkInputStream(ChunkInfo chunkInfo,
BlockID blockId,
XceiverClientSpi xceiverClient,
boolean verifyChecksum,
byte[] data) {
super(chunkInfo, blockId, xceiverClient, verifyChecksum);
chunkData = data;
}
@Override
protected ByteString readChunk(ChunkInfo readChunkInfo) {
ByteString byteString = ByteString.copyFrom(chunkData,
(int) readChunkInfo.getOffset(),
(int) readChunkInfo.getLen());
readByteBuffers.add(byteString);
return byteString;
}
@Override
protected void checkOpen() {
// No action needed
}
}
/**
* Match readData with the chunkData byte-wise.
* @param readData Data read through ChunkInputStream
* @param inputDataStartIndex first index (inclusive) in chunkData to compare
* with read data
* @param length the number of bytes of data to match starting from
* inputDataStartIndex
*/
private void matchWithInputData(byte[] readData, int inputDataStartIndex,
int length) {
for (int i = inputDataStartIndex; i < inputDataStartIndex + length; i++) {
Assert.assertEquals(chunkData[i], readData[i - inputDataStartIndex]);
}
}
/**
* Seek to a position and verify through getPos().
*/
private void seekAndVerify(int pos) throws Exception {
chunkStream.seek(pos);
Assert.assertEquals("Current position of buffer does not match with the " +
"seeked position", pos, chunkStream.getPos());
}
@Test
public void testFullChunkRead() throws Exception {
byte[] b = new byte[CHUNK_SIZE];
chunkStream.read(b, 0, CHUNK_SIZE);
matchWithInputData(b, 0, CHUNK_SIZE);
}
@Test
public void testPartialChunkRead() throws Exception {
int len = CHUNK_SIZE / 2;
byte[] b = new byte[len];
chunkStream.read(b, 0, len);
matchWithInputData(b, 0, len);
// To read chunk data from index 0 to 49 (len = 50), we need to read
// chunk from offset 0 to 60 as the checksum boundary is at every 20
// bytes. Verify that 60 bytes of chunk data are read and stored in the
// buffers.
matchWithInputData(chunkStream.readByteBuffers.get(0).toByteArray(),
0, 60);
}
@Test
public void testSeek() throws Exception {
seekAndVerify(0);
try {
seekAndVerify(CHUNK_SIZE);
Assert.fail("Seeking to Chunk Length should fail.");
} catch (EOFException e) {
GenericTestUtils.assertExceptionContains("EOF encountered at pos: "
+ CHUNK_SIZE + " for chunk: " + CHUNK_NAME, e);
}
// Seek before read should update the ChunkInputStream#chunkPosition
seekAndVerify(25);
Assert.assertEquals(25, chunkStream.getChunkPosition());
// Read from the seeked position.
// Reading from index 25 to 54 should result in the ChunkInputStream
// copying chunk data from index 20 to 59 into the buffers (checksum
// boundaries).
byte[] b = new byte[30];
chunkStream.read(b, 0, 30);
matchWithInputData(b, 25, 30);
matchWithInputData(chunkStream.readByteBuffers.get(0).toByteArray(),
20, 40);
// After read, the position of the chunkStream is evaluated from the
// buffers and the chunkPosition should be reset to -1.
Assert.assertEquals(-1, chunkStream.getChunkPosition());
// Seek to a position within the current buffers. Current buffers contain
// data from index 20 to 59. ChunkPosition should still not be used to
// set the position.
seekAndVerify(35);
Assert.assertEquals(-1, chunkStream.getChunkPosition());
// Seek to a position outside the current buffers. In this case, the
// chunkPosition should be updated to the seeked position.
seekAndVerify(75);
Assert.assertEquals(75, chunkStream.getChunkPosition());
}
@Test
public void testSeekAndRead() throws Exception {
// Seek to a position and read data
seekAndVerify(50);
byte[] b1 = new byte[20];
chunkStream.read(b1, 0, 20);
matchWithInputData(b1, 50, 20);
// Next read should start from the position of the last read + 1 i.e. 70
byte[] b2 = new byte[20];
chunkStream.read(b2, 0, 20);
matchWithInputData(b2, 70, 20);
}
}

21
hadoop-hdds/client/src/test/java/org/apache/hadoop/hdds/scm/storage/package-info.java
View File

@ -1,21 +0,0 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations under
* the License.
*/
/**
* This package contains Ozone InputStream related tests.
*/
package org.apache.hadoop.hdds.scm.storage;

33
hadoop-hdds/common/dev-support/findbugsExcludeFile.xml
View File

@ -1,33 +0,0 @@
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<FindBugsFilter>
<Match>
<Package name="org.apache.hadoop.hdds.protocol.proto"/>
</Match>
<Match>
<Package name="org.apache.hadoop.hdds.protocol.datanode.proto"/>
</Match>
<Match>
<Class name="org.apache.hadoop.hdds.cli.GenericCli"></Class>
<Bug pattern="DM_EXIT" />
</Match>
<Match>
<Class name="org.apache.hadoop.ozone.common.ChecksumByteBuffer$CrcIntTable" />
<Method name="update" />
<Bug pattern="SF_SWITCH_FALLTHROUGH,SF_SWITCH_NO_DEFAULT" />
</Match>
</FindBugsFilter>

285
hadoop-hdds/common/pom.xml
View File

@ -1,285 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. See accompanying LICENSE file.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdds</artifactId>
<version>0.5.0-SNAPSHOT</version>
</parent>
<artifactId>hadoop-hdds-common</artifactId>
<version>0.5.0-SNAPSHOT</version>
<description>Apache Hadoop Distributed Data Store Common</description>
<name>Apache Hadoop HDDS Common</name>
<packaging>jar</packaging>
<properties>
<hdds.version>0.5.0-SNAPSHOT</hdds.version>
<log4j2.version>2.11.0</log4j2.version>
<disruptor.version>3.4.2</disruptor.version>
<declared.hdds.version>${hdds.version}</declared.hdds.version>
</properties>
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdds-config</artifactId>
</dependency>
<dependency>
<groupId>javax.annotation</groupId>
<artifactId>javax.annotation-api</artifactId>
<version>1.2</version>
</dependency>
<dependency>
<groupId>org.fusesource.leveldbjni</groupId>
<artifactId>leveldbjni-all</artifactId>
</dependency>
<dependency>
<artifactId>ratis-server</artifactId>
<groupId>org.apache.ratis</groupId>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</exclusion>
<exclusion>
<groupId>io.dropwizard.metrics</groupId>
<artifactId>metrics-core</artifactId>
</exclusion>
<exclusion>
<groupId>org.bouncycastle</groupId>
<artifactId>bcprov-jdk15on</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<artifactId>ratis-netty</artifactId>
<groupId>org.apache.ratis</groupId>
</dependency>
<dependency>
<artifactId>ratis-grpc</artifactId>
<groupId>org.apache.ratis</groupId>
</dependency>
<dependency>
<groupId>com.google.errorprone</groupId>
<artifactId>error_prone_annotations</artifactId>
<version>2.2.0</version>
<optional>true</optional>
</dependency>
<dependency>
<groupId>org.rocksdb</groupId>
<artifactId>rocksdbjni</artifactId>
<version>6.0.1</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<scope>test</scope>
<type>test-jar</type>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-api</artifactId>
<version>${log4j2.version}</version>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-core</artifactId>
<version>${log4j2.version}</version>
</dependency>
<dependency>
<groupId>com.lmax</groupId>
<artifactId>disruptor</artifactId>
<version>${disruptor.version}</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-pool2</artifactId>
<version>2.6.0</version>
</dependency>
<dependency>
<groupId>org.bouncycastle</groupId>
<artifactId>bcpkix-jdk15on</artifactId>
<version>${bouncycastle.version}</version>
</dependency>
<!-- https://mvnrepository.com/artifact/commons-validator/commons-validator -->
<dependency>
<groupId>commons-validator</groupId>
<artifactId>commons-validator</artifactId>
<version>1.6</version>
</dependency>
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter-api</artifactId>
</dependency>
<dependency>
<groupId>io.jaegertracing</groupId>
<artifactId>jaeger-client</artifactId>
<version>${jaeger.version}</version>
</dependency>
<dependency>
<groupId>io.opentracing</groupId>
<artifactId>opentracing-util</artifactId>
<version>0.31.0</version>
</dependency>
<dependency>
<groupId>org.yaml</groupId>
<artifactId>snakeyaml</artifactId>
<version>1.16</version>
</dependency>
</dependencies>
<build>
<resources>
<resource>
<directory>${basedir}/src/main/resources</directory>
<excludes>
<exclude>hdds-version-info.properties</exclude>
</excludes>
<filtering>false</filtering>
</resource>
<resource>
<directory>${basedir}/src/main/resources</directory>
<includes>
<include>hdds-version-info.properties</include>
</includes>
<filtering>true</filtering>
</resource>
</resources>
<extensions>
<extension>
<groupId>kr.motd.maven</groupId>
<artifactId>os-maven-plugin</artifactId>
<version>${os-maven-plugin.version}</version>
</extension>
</extensions>
<plugins>
<plugin>
<groupId>org.xolstice.maven.plugins</groupId>
<artifactId>protobuf-maven-plugin</artifactId>
<version>${protobuf-maven-plugin.version}</version>
<extensions>true</extensions>
<configuration>
<protocArtifact>
com.google.protobuf:protoc:${protobuf-compile.version}:exe:${os.detected.classifier}
</protocArtifact>
<protoSourceRoot>${basedir}/src/main/proto/</protoSourceRoot>
<includes>
<include>DatanodeContainerProtocol.proto</include>
</includes>
<outputDirectory>target/generated-sources/java</outputDirectory>
<clearOutputDirectory>false</clearOutputDirectory>
</configuration>
<executions>
<execution>
<id>compile-protoc</id>
<goals>
<goal>compile</goal>
<goal>test-compile</goal>
<goal>compile-custom</goal>
<goal>test-compile-custom</goal>
</goals>
<configuration>
<pluginId>grpc-java</pluginId>
<pluginArtifact>
io.grpc:protoc-gen-grpc-java:${grpc.version}:exe:${os.detected.classifier}
</pluginArtifact>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-antrun-plugin</artifactId>
<executions>
<execution>
<phase>generate-sources</phase>
<configuration>
<tasks>
<replace token="com.google.protobuf" value="org.apache.ratis.thirdparty.com.google.protobuf"
dir="target/generated-sources/java/org/apache/hadoop/hdds/protocol/datanode/proto">
</replace>
<replace token="io.grpc" value="org.apache.ratis.thirdparty.io.grpc"
dir="target/generated-sources/java/org/apache/hadoop/hdds/protocol/datanode/proto">
</replace>
</tasks>
</configuration>
<goals>
<goal>run</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-maven-plugins</artifactId>
<executions>
<execution>
<id>version-info</id>
<phase>generate-resources</phase>
<goals>
<goal>version-info</goal>
</goals>
<configuration>
<source>
<directory>${basedir}/../</directory>
<includes>
<include>*/src/main/java/**/*.java</include>
<include>*/src/main/proto/*.proto</include>
</includes>
</source>
</configuration>
</execution>
<execution>
<id>compile-protoc</id>
<goals>
<goal>protoc</goal>
</goals>
<configuration>
<protocVersion>${protobuf.version}</protocVersion>
<protocCommand>${protoc.path}</protocCommand>
<imports>
<param>${basedir}/src/main/proto</param>
</imports>
<source>
<directory>${basedir}/src/main/proto</directory>
<includes>
<include>StorageContainerLocationProtocol.proto</include>
<include>hdds.proto</include>
<include>ScmBlockLocationProtocol.proto</include>
<include>SCMSecurityProtocol.proto</include>
</includes>
</source>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>com.github.spotbugs</groupId>
<artifactId>spotbugs-maven-plugin</artifactId>
<configuration>
<excludeFilterFile>${basedir}/dev-support/findbugsExcludeFile.xml</excludeFilterFile>
</configuration>
</plugin>
</plugins>
</build>
</project>

317
hadoop-hdds/common/src/main/bin/hadoop-config.cmd
View File

@ -1,317 +0,0 @@
@echo off
@rem Licensed to the Apache Software Foundation (ASF) under one or more
@rem contributor license agreements. See the NOTICE file distributed with
@rem this work for additional information regarding copyright ownership.
@rem The ASF licenses this file to You under the Apache License, Version 2.0
@rem (the "License"); you may not use this file except in compliance with
@rem the License. You may obtain a copy of the License at
@rem
@rem http://www.apache.org/licenses/LICENSE-2.0
@rem
@rem Unless required by applicable law or agreed to in writing, software
@rem distributed under the License is distributed on an "AS IS" BASIS,
@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@rem See the License for the specific language governing permissions and
@rem limitations under the License.
@rem included in all the hadoop scripts with source command
@rem should not be executable directly
@rem also should not be passed any arguments, since we need original %*
if not defined HADOOP_COMMON_DIR (
set HADOOP_COMMON_DIR=share\hadoop\common
)
if not defined HADOOP_COMMON_LIB_JARS_DIR (
set HADOOP_COMMON_LIB_JARS_DIR=share\hadoop\common\lib
)
if not defined HADOOP_COMMON_LIB_NATIVE_DIR (
set HADOOP_COMMON_LIB_NATIVE_DIR=lib\native
)
if not defined HDFS_DIR (
set HDFS_DIR=share\hadoop\hdfs
)
if not defined HDFS_LIB_JARS_DIR (
set HDFS_LIB_JARS_DIR=share\hadoop\hdfs\lib
)
if not defined YARN_DIR (
set YARN_DIR=share\hadoop\yarn
)
if not defined YARN_LIB_JARS_DIR (
set YARN_LIB_JARS_DIR=share\hadoop\yarn\lib
)
if not defined MAPRED_DIR (
set MAPRED_DIR=share\hadoop\mapreduce
)
if not defined MAPRED_LIB_JARS_DIR (
set MAPRED_LIB_JARS_DIR=share\hadoop\mapreduce\lib
)
@rem the root of the Hadoop installation
set HADOOP_HOME=%~dp0
for %%i in (%HADOOP_HOME%.) do (
set HADOOP_HOME=%%~dpi
)
if "%HADOOP_HOME:~-1%" == "\" (
set HADOOP_HOME=%HADOOP_HOME:~0,-1%
)
if not exist %HADOOP_HOME%\share\hadoop\common\hadoop-common-*.jar (
@echo +================================================================+
@echo ^| Error: HADOOP_HOME is not set correctly ^|
@echo +----------------------------------------------------------------+
@echo ^| Please set your HADOOP_HOME variable to the absolute path of ^|
@echo ^| the directory that contains the hadoop distribution ^|
@echo +================================================================+
exit /b 1
)
if not defined HADOOP_CONF_DIR (
set HADOOP_CONF_DIR=%HADOOP_HOME%\etc\hadoop
)
@rem
@rem Allow alternate conf dir location.
@rem
if "%1" == "--config" (
set HADOOP_CONF_DIR=%2
shift
shift
)
@rem
@rem check to see it is specified whether to use the workers or the
@rem masters file
@rem
if "%1" == "--hosts" (
set HADOOP_WORKERS=%HADOOP_CONF_DIR%\%2
shift
shift
)
@rem
@rem Set log level. Default to INFO.
@rem
if "%1" == "--loglevel" (
set HADOOP_LOGLEVEL=%2
shift
shift
)
if exist %HADOOP_CONF_DIR%\hadoop-env.cmd (
call %HADOOP_CONF_DIR%\hadoop-env.cmd
)
@rem
@rem setup java environment variables
@rem
if not defined JAVA_HOME (
echo Error: JAVA_HOME is not set.
goto :eof
)
if not exist %JAVA_HOME%\bin\java.exe (
echo Error: JAVA_HOME is incorrectly set.
echo Please update %HADOOP_CONF_DIR%\hadoop-env.cmd
goto :eof
)
set JAVA=%JAVA_HOME%\bin\java
@rem some Java parameters
set JAVA_HEAP_MAX=-Xmx1000m
@rem
@rem check envvars which might override default args
@rem
if defined HADOOP_HEAPSIZE (
set JAVA_HEAP_MAX=-Xmx%HADOOP_HEAPSIZE%m
)
@rem
@rem CLASSPATH initially contains %HADOOP_CONF_DIR%
@rem
set CLASSPATH=%HADOOP_CONF_DIR%
if not defined HADOOP_COMMON_HOME (
if exist %HADOOP_HOME%\share\hadoop\common (
set HADOOP_COMMON_HOME=%HADOOP_HOME%
)
)
@rem
@rem for releases, add core hadoop jar & webapps to CLASSPATH
@rem
if exist %HADOOP_COMMON_HOME%\%HADOOP_COMMON_DIR%\webapps (
set CLASSPATH=!CLASSPATH!;%HADOOP_COMMON_HOME%\%HADOOP_COMMON_DIR%
)
if exist %HADOOP_COMMON_HOME%\%HADOOP_COMMON_LIB_JARS_DIR% (
set CLASSPATH=!CLASSPATH!;%HADOOP_COMMON_HOME%\%HADOOP_COMMON_LIB_JARS_DIR%\*
)
set CLASSPATH=!CLASSPATH!;%HADOOP_COMMON_HOME%\%HADOOP_COMMON_DIR%\*
@rem
@rem default log directory % file
@rem
if not defined HADOOP_LOG_DIR (
set HADOOP_LOG_DIR=%HADOOP_HOME%\logs
)
if not defined HADOOP_LOGFILE (
set HADOOP_LOGFILE=hadoop.log
)
if not defined HADOOP_LOGLEVEL (
set HADOOP_LOGLEVEL=INFO
)
if not defined HADOOP_ROOT_LOGGER (
set HADOOP_ROOT_LOGGER=%HADOOP_LOGLEVEL%,console
)
@rem
@rem default policy file for service-level authorization
@rem
if not defined HADOOP_POLICYFILE (
set HADOOP_POLICYFILE=hadoop-policy.xml
)
@rem
@rem Determine the JAVA_PLATFORM
@rem
for /f "delims=" %%A in ('%JAVA% -Xmx32m %HADOOP_JAVA_PLATFORM_OPTS% -classpath "%CLASSPATH%" org.apache.hadoop.util.PlatformName') do set JAVA_PLATFORM=%%A
@rem replace space with underscore
set JAVA_PLATFORM=%JAVA_PLATFORM: =_%
@rem
@rem setup 'java.library.path' for native hadoop code if necessary
@rem
@rem Check if we're running hadoop directly from the build
if exist %HADOOP_COMMON_HOME%\target\bin (
if defined JAVA_LIBRARY_PATH (
set JAVA_LIBRARY_PATH=%JAVA_LIBRARY_PATH%;%HADOOP_COMMON_HOME%\target\bin
) else (
set JAVA_LIBRARY_PATH=%HADOOP_COMMON_HOME%\target\bin
)
)
@rem For the distro case, check the bin folder
if exist %HADOOP_COMMON_HOME%\bin (
if defined JAVA_LIBRARY_PATH (
set JAVA_LIBRARY_PATH=%JAVA_LIBRARY_PATH%;%HADOOP_COMMON_HOME%\bin
) else (
set JAVA_LIBRARY_PATH=%HADOOP_COMMON_HOME%\bin
)
)
@rem
@rem setup a default TOOL_PATH
@rem
set TOOL_PATH=%HADOOP_HOME%\share\hadoop\tools\lib\*
set HADOOP_OPTS=%HADOOP_OPTS% -Dhadoop.log.dir=%HADOOP_LOG_DIR%
set HADOOP_OPTS=%HADOOP_OPTS% -Dhadoop.log.file=%HADOOP_LOGFILE%
set HADOOP_OPTS=%HADOOP_OPTS% -Dhadoop.home.dir=%HADOOP_HOME%
set HADOOP_OPTS=%HADOOP_OPTS% -Dhadoop.id.str=%HADOOP_IDENT_STRING%
set HADOOP_OPTS=%HADOOP_OPTS% -Dhadoop.root.logger=%HADOOP_ROOT_LOGGER%
if defined JAVA_LIBRARY_PATH (
set HADOOP_OPTS=%HADOOP_OPTS% -Djava.library.path=%JAVA_LIBRARY_PATH%
)
set HADOOP_OPTS=%HADOOP_OPTS% -Dhadoop.policy.file=%HADOOP_POLICYFILE%
@rem
@rem Disable ipv6 as it can cause issues
@rem
set HADOOP_OPTS=%HADOOP_OPTS% -Djava.net.preferIPv4Stack=true
@rem
@rem put hdfs in classpath if present
@rem
if not defined HADOOP_HDFS_HOME (
if exist %HADOOP_HOME%\%HDFS_DIR% (
set HADOOP_HDFS_HOME=%HADOOP_HOME%
)
)
if exist %HADOOP_HDFS_HOME%\%HDFS_DIR%\webapps (
set CLASSPATH=!CLASSPATH!;%HADOOP_HDFS_HOME%\%HDFS_DIR%
)
if exist %HADOOP_HDFS_HOME%\%HDFS_LIB_JARS_DIR% (
set CLASSPATH=!CLASSPATH!;%HADOOP_HDFS_HOME%\%HDFS_LIB_JARS_DIR%\*
)
set CLASSPATH=!CLASSPATH!;%HADOOP_HDFS_HOME%\%HDFS_DIR%\*
@rem
@rem put yarn in classpath if present
@rem
if not defined HADOOP_YARN_HOME (
if exist %HADOOP_HOME%\%YARN_DIR% (
set HADOOP_YARN_HOME=%HADOOP_HOME%
)
)
if exist %HADOOP_YARN_HOME%\%YARN_DIR%\webapps (
set CLASSPATH=!CLASSPATH!;%HADOOP_YARN_HOME%\%YARN_DIR%
)
if exist %HADOOP_YARN_HOME%\%YARN_LIB_JARS_DIR% (
set CLASSPATH=!CLASSPATH!;%HADOOP_YARN_HOME%\%YARN_LIB_JARS_DIR%\*
)
set CLASSPATH=!CLASSPATH!;%HADOOP_YARN_HOME%\%YARN_DIR%\*
@rem
@rem put mapred in classpath if present AND different from YARN
@rem
if not defined HADOOP_MAPRED_HOME (
if exist %HADOOP_HOME%\%MAPRED_DIR% (
set HADOOP_MAPRED_HOME=%HADOOP_HOME%
)
)
if not "%HADOOP_MAPRED_HOME%\%MAPRED_DIR%" == "%HADOOP_YARN_HOME%\%YARN_DIR%" (
if exist %HADOOP_MAPRED_HOME%\%MAPRED_DIR%\webapps (
set CLASSPATH=!CLASSPATH!;%HADOOP_MAPRED_HOME%\%MAPRED_DIR%
)
if exist %HADOOP_MAPRED_HOME%\%MAPRED_LIB_JARS_DIR% (
set CLASSPATH=!CLASSPATH!;%HADOOP_MAPRED_HOME%\%MAPRED_LIB_JARS_DIR%\*
)
set CLASSPATH=!CLASSPATH!;%HADOOP_MAPRED_HOME%\%MAPRED_DIR%\*
)
@rem
@rem add user-specified CLASSPATH last
@rem
if defined HADOOP_CLASSPATH (
if not defined HADOOP_USE_CLIENT_CLASSLOADER (
if defined HADOOP_USER_CLASSPATH_FIRST (
set CLASSPATH=%HADOOP_CLASSPATH%;%CLASSPATH%;
) else (
set CLASSPATH=%CLASSPATH%;%HADOOP_CLASSPATH%;
)
)
)
:eof

165
hadoop-hdds/common/src/main/bin/hadoop-config.sh
View File

@ -1,165 +0,0 @@
#!/usr/bin/env bash
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
####
# IMPORTANT
####
## The hadoop-config.sh tends to get executed by non-Hadoop scripts.
## Those parts expect this script to parse/manipulate $@. In order
## to maintain backward compatibility, this means a surprising
## lack of functions for bits that would be much better off in
## a function.
##
## In other words, yes, there is some bad things happen here and
## unless we break the rest of the ecosystem, we can't change it. :(
# included in all the hadoop scripts with source command
# should not be executable directly
# also should not be passed any arguments, since we need original $*
#
# after doing more config, caller should also exec finalize
# function to finish last minute/default configs for
# settings that might be different between daemons & interactive
# you must be this high to ride the ride
if [[ -z "${BASH_VERSINFO[0]}" ]] \
|| [[ "${BASH_VERSINFO[0]}" -lt 3 ]] \
|| [[ "${BASH_VERSINFO[0]}" -eq 3 && "${BASH_VERSINFO[1]}" -lt 2 ]]; then
echo "bash v3.2+ is required. Sorry."
exit 1
fi
# In order to get partially bootstrapped, we need to figure out where
# we are located. Chances are good that our caller has already done
# this work for us, but just in case...
if [[ -z "${HADOOP_LIBEXEC_DIR}" ]]; then
_hadoop_common_this="${BASH_SOURCE-$0}"
HADOOP_LIBEXEC_DIR=$(cd -P -- "$(dirname -- "${_hadoop_common_this}")" >/dev/null && pwd -P)
fi
# get our functions defined for usage later
if [[ -n "${HADOOP_COMMON_HOME}" ]] &&
[[ -e "${HADOOP_COMMON_HOME}/libexec/hadoop-functions.sh" ]]; then
# shellcheck source=./hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
. "${HADOOP_COMMON_HOME}/libexec/hadoop-functions.sh"
elif [[ -e "${HADOOP_LIBEXEC_DIR}/hadoop-functions.sh" ]]; then
# shellcheck source=./hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
. "${HADOOP_LIBEXEC_DIR}/hadoop-functions.sh"
else
echo "ERROR: Unable to exec ${HADOOP_LIBEXEC_DIR}/hadoop-functions.sh." 1>&2
exit 1
fi
hadoop_deprecate_envvar HADOOP_PREFIX HADOOP_HOME
# allow overrides of the above and pre-defines of the below
if [[ -n "${HADOOP_COMMON_HOME}" ]] &&
[[ -e "${HADOOP_COMMON_HOME}/libexec/hadoop-layout.sh" ]]; then
# shellcheck source=./hadoop-common-project/hadoop-common/src/main/bin/hadoop-layout.sh.example
. "${HADOOP_COMMON_HOME}/libexec/hadoop-layout.sh"
elif [[ -e "${HADOOP_LIBEXEC_DIR}/hadoop-layout.sh" ]]; then
# shellcheck source=./hadoop-common-project/hadoop-common/src/main/bin/hadoop-layout.sh.example
. "${HADOOP_LIBEXEC_DIR}/hadoop-layout.sh"
fi
#
# IMPORTANT! We are not executing user provided code yet!
#
# Let's go! Base definitions so we can move forward
hadoop_bootstrap
# let's find our conf.
#
# first, check and process params passed to us
# we process this in-line so that we can directly modify $@
# if something downstream is processing that directly,
# we need to make sure our params have been ripped out
# note that we do many of them here for various utilities.
# this provides consistency and forces a more consistent
# user experience
# save these off in case our caller needs them
# shellcheck disable=SC2034
HADOOP_USER_PARAMS=("$@")
hadoop_parse_args "$@"
shift "${HADOOP_PARSE_COUNTER}"
#
# Setup the base-line environment
#
hadoop_find_confdir
hadoop_exec_hadoopenv
hadoop_import_shellprofiles
hadoop_exec_userfuncs
#
# IMPORTANT! User provided code is now available!
#
hadoop_exec_user_hadoopenv
hadoop_verify_confdir
hadoop_deprecate_envvar HADOOP_SLAVES HADOOP_WORKERS
hadoop_deprecate_envvar HADOOP_SLAVE_NAMES HADOOP_WORKER_NAMES
hadoop_deprecate_envvar HADOOP_SLAVE_SLEEP HADOOP_WORKER_SLEEP
# do all the OS-specific startup bits here
# this allows us to get a decent JAVA_HOME,
# call crle for LD_LIBRARY_PATH, etc.
hadoop_os_tricks
hadoop_java_setup
hadoop_basic_init
# inject any sub-project overrides, defaults, etc.
if declare -F hadoop_subproject_init >/dev/null ; then
hadoop_subproject_init
fi
hadoop_shellprofiles_init
# get the native libs in there pretty quick
hadoop_add_javalibpath "${HADOOP_HOME}/build/native"
hadoop_add_javalibpath "${HADOOP_HOME}/${HADOOP_COMMON_LIB_NATIVE_DIR}"
hadoop_shellprofiles_nativelib
# get the basic java class path for these subprojects
# in as quickly as possible since other stuff
# will definitely depend upon it.
hadoop_add_common_to_classpath
hadoop_shellprofiles_classpath
# user API commands can now be run since the runtime
# environment has been configured
hadoop_exec_hadooprc
#
# backwards compatibility. new stuff should
# call this when they are ready
#
if [[ -z "${HADOOP_NEW_CONFIG}" ]]; then
hadoop_finalize
fi

77
hadoop-hdds/common/src/main/bin/hadoop-daemons.sh
View File

@ -1,77 +0,0 @@
#!/usr/bin/env bash
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Run a Hadoop command on all slave hosts.
function hadoop_usage
{
echo "Usage: hadoop-daemons.sh [--config confdir] [--hosts hostlistfile] (start|stop|status) <hadoop-command> <args...>"
}
this="${BASH_SOURCE-$0}"
bin=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P)
# let's locate libexec...
if [[ -n "${HADOOP_HOME}" ]]; then
HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_HOME}/libexec"
else
HADOOP_DEFAULT_LIBEXEC_DIR="${bin}/../libexec"
fi
HADOOP_LIBEXEC_DIR="${HADOOP_LIBEXEC_DIR:-$HADOOP_DEFAULT_LIBEXEC_DIR}"
# shellcheck disable=SC2034
HADOOP_NEW_CONFIG=true
if [[ -f "${HADOOP_LIBEXEC_DIR}/hdfs-config.sh" ]]; then
. "${HADOOP_LIBEXEC_DIR}/hdfs-config.sh"
else
echo "ERROR: Cannot execute ${HADOOP_LIBEXEC_DIR}/hdfs-config.sh." 2>&1
exit 1
fi
if [[ $# = 0 ]]; then
hadoop_exit_with_usage 1
fi
daemonmode=$1
shift
if [[ -z "${HADOOP_HDFS_HOME}" ]]; then
hdfsscript="${HADOOP_HOME}/bin/hdfs"
else
hdfsscript="${HADOOP_HDFS_HOME}/bin/hdfs"
fi
hadoop_error "WARNING: Use of this script to ${daemonmode} HDFS daemons is deprecated."
hadoop_error "WARNING: Attempting to execute replacement \"hdfs --workers --daemon ${daemonmode}\" instead."
#
# Original input was usually:
# hadoop-daemons.sh (shell options) (start|stop) (datanode|...) (daemon options)
# we're going to turn this into
# hdfs --workers --daemon (start|stop) (rest of options)
#
for (( i = 0; i < ${#HADOOP_USER_PARAMS[@]}; i++ ))
do
if [[ "${HADOOP_USER_PARAMS[$i]}" =~ ^start$ ]] ||
[[ "${HADOOP_USER_PARAMS[$i]}" =~ ^stop$ ]] ||
[[ "${HADOOP_USER_PARAMS[$i]}" =~ ^status$ ]]; then
unset HADOOP_USER_PARAMS[$i]
fi
done
${hdfsscript} --workers --daemon "${daemonmode}" "${HADOOP_USER_PARAMS[@]}"

2732
hadoop-hdds/common/src/main/bin/hadoop-functions.sh
View File

File diff suppressed because it is too large Load Diff

59
hadoop-hdds/common/src/main/bin/workers.sh
View File

@ -1,59 +0,0 @@
#!/usr/bin/env bash
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Run a shell command on all worker hosts.
#
# Environment Variables
#
# HADOOP_WORKERS File naming remote hosts.
# Default is ${HADOOP_CONF_DIR}/workers.
# HADOOP_CONF_DIR Alternate conf dir. Default is ${HADOOP_HOME}/conf.
# HADOOP_WORKER_SLEEP Seconds to sleep between spawning remote commands.
# HADOOP_SSH_OPTS Options passed to ssh when running remote commands.
##
function hadoop_usage
{
echo "Usage: workers.sh [--config confdir] command..."
}
# let's locate libexec...
if [[ -n "${HADOOP_HOME}" ]]; then
HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_HOME}/libexec"
else
this="${BASH_SOURCE-$0}"
bin=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P)
HADOOP_DEFAULT_LIBEXEC_DIR="${bin}/../libexec"
fi
HADOOP_LIBEXEC_DIR="${HADOOP_LIBEXEC_DIR:-$HADOOP_DEFAULT_LIBEXEC_DIR}"
# shellcheck disable=SC2034
HADOOP_NEW_CONFIG=true
if [[ -f "${HADOOP_LIBEXEC_DIR}/hadoop-config.sh" ]]; then
. "${HADOOP_LIBEXEC_DIR}/hadoop-config.sh"
else
echo "ERROR: Cannot execute ${HADOOP_LIBEXEC_DIR}/hadoop-config.sh." 2>&1
exit 1
fi
# if no args specified, show usage
if [[ $# -le 0 ]]; then
hadoop_exit_with_usage 1
fi
hadoop_connect_to_hosts "$@"

20
hadoop-hdds/common/src/main/conf/core-site.xml
View File

@ -1,20 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!--
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. See accompanying LICENSE file.
-->
<!-- Put site-specific property overrides in this file. -->
<configuration>
</configuration>

90
hadoop-hdds/common/src/main/conf/hadoop-env.cmd
View File

@ -1,90 +0,0 @@
@echo off
@rem Licensed to the Apache Software Foundation (ASF) under one or more
@rem contributor license agreements. See the NOTICE file distributed with
@rem this work for additional information regarding copyright ownership.
@rem The ASF licenses this file to You under the Apache License, Version 2.0
@rem (the "License"); you may not use this file except in compliance with
@rem the License. You may obtain a copy of the License at
@rem
@rem http://www.apache.org/licenses/LICENSE-2.0
@rem
@rem Unless required by applicable law or agreed to in writing, software
@rem distributed under the License is distributed on an "AS IS" BASIS,
@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@rem See the License for the specific language governing permissions and
@rem limitations under the License.
@rem Set Hadoop-specific environment variables here.
@rem The only required environment variable is JAVA_HOME. All others are
@rem optional. When running a distributed configuration it is best to
@rem set JAVA_HOME in this file, so that it is correctly defined on
@rem remote nodes.
@rem The java implementation to use. Required.
set JAVA_HOME=%JAVA_HOME%
@rem The jsvc implementation to use. Jsvc is required to run secure datanodes.
@rem set JSVC_HOME=%JSVC_HOME%
@rem set HADOOP_CONF_DIR=
@rem Extra Java CLASSPATH elements. Automatically insert capacity-scheduler.
if exist %HADOOP_HOME%\contrib\capacity-scheduler (
if not defined HADOOP_CLASSPATH (
set HADOOP_CLASSPATH=%HADOOP_HOME%\contrib\capacity-scheduler\*.jar
) else (
set HADOOP_CLASSPATH=%HADOOP_CLASSPATH%;%HADOOP_HOME%\contrib\capacity-scheduler\*.jar
)
)
@rem The maximum amount of heap to use, in MB. Default is 1000.
@rem set HADOOP_HEAPSIZE=
@rem set HADOOP_NAMENODE_INIT_HEAPSIZE=""
@rem Extra Java runtime options. Empty by default.
@rem set HADOOP_OPTS=%HADOOP_OPTS% -Djava.net.preferIPv4Stack=true
@rem Command specific options appended to HADOOP_OPTS when specified
if not defined HADOOP_SECURITY_LOGGER (
set HADOOP_SECURITY_LOGGER=INFO,RFAS
)
if not defined HDFS_AUDIT_LOGGER (
set HDFS_AUDIT_LOGGER=INFO,NullAppender
)
set HADOOP_NAMENODE_OPTS=-Dhadoop.security.logger=%HADOOP_SECURITY_LOGGER% -Dhdfs.audit.logger=%HDFS_AUDIT_LOGGER% %HADOOP_NAMENODE_OPTS%
set HADOOP_DATANODE_OPTS=-Dhadoop.security.logger=ERROR,RFAS %HADOOP_DATANODE_OPTS%
set HADOOP_SECONDARYNAMENODE_OPTS=-Dhadoop.security.logger=%HADOOP_SECURITY_LOGGER% -Dhdfs.audit.logger=%HDFS_AUDIT_LOGGER% %HADOOP_SECONDARYNAMENODE_OPTS%
@rem The following applies to multiple commands (fs, dfs, fsck, distcp etc)
set HADOOP_CLIENT_OPTS=-Xmx512m %HADOOP_CLIENT_OPTS%
@rem set HADOOP_JAVA_PLATFORM_OPTS="-XX:-UsePerfData %HADOOP_JAVA_PLATFORM_OPTS%"
@rem On secure datanodes, user to run the datanode as after dropping privileges
set HADOOP_SECURE_DN_USER=%HADOOP_SECURE_DN_USER%
@rem Where log files are stored. %HADOOP_HOME%/logs by default.
@rem set HADOOP_LOG_DIR=%HADOOP_LOG_DIR%\%USERNAME%
@rem Where log files are stored in the secure data environment.
set HADOOP_SECURE_DN_LOG_DIR=%HADOOP_LOG_DIR%\%HADOOP_HDFS_USER%
@rem
@rem Router-based HDFS Federation specific parameters
@rem Specify the JVM options to be used when starting the RBF Routers.
@rem These options will be appended to the options specified as HADOOP_OPTS
@rem and therefore may override any similar flags set in HADOOP_OPTS
@rem
@rem set HADOOP_DFSROUTER_OPTS=""
@rem
@rem The directory where pid files are stored. /tmp by default.
@rem NOTE: this should be set to a directory that can only be written to by
@rem the user that will run the hadoop daemons. Otherwise there is the
@rem potential for a symlink attack.
set HADOOP_PID_DIR=%HADOOP_PID_DIR%
set HADOOP_SECURE_DN_PID_DIR=%HADOOP_PID_DIR%
@rem A string representing this instance of hadoop. %USERNAME% by default.
set HADOOP_IDENT_STRING=%USERNAME%

439
hadoop-hdds/common/src/main/conf/hadoop-env.sh
View File

@ -1,439 +0,0 @@
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Set Hadoop-specific environment variables here.
##
## THIS FILE ACTS AS THE MASTER FILE FOR ALL HADOOP PROJECTS.
## SETTINGS HERE WILL BE READ BY ALL HADOOP COMMANDS. THEREFORE,
## ONE CAN USE THIS FILE TO SET YARN, HDFS, AND MAPREDUCE
## CONFIGURATION OPTIONS INSTEAD OF xxx-env.sh.
##
## Precedence rules:
##
## {yarn-env.sh|hdfs-env.sh} > hadoop-env.sh > hard-coded defaults
##
## {YARN_xyz|HDFS_xyz} > HADOOP_xyz > hard-coded defaults
##
# Many of the options here are built from the perspective that users
# may want to provide OVERWRITING values on the command line.
# For example:
#
# JAVA_HOME=/usr/java/testing hdfs dfs -ls
#
# Therefore, the vast majority (BUT NOT ALL!) of these defaults
# are configured for substitution and not append. If append
# is preferable, modify this file accordingly.
###
# Generic settings for HADOOP
###
# Technically, the only required environment variable is JAVA_HOME.
# All others are optional. However, the defaults are probably not
# preferred. Many sites configure these options outside of Hadoop,
# such as in /etc/profile.d
# The java implementation to use. By default, this environment
# variable is REQUIRED on ALL platforms except OS X!
# export JAVA_HOME=
# Location of Hadoop. By default, Hadoop will attempt to determine
# this location based upon its execution path.
# export HADOOP_HOME=
# Location of Hadoop's configuration information. i.e., where this
# file is living. If this is not defined, Hadoop will attempt to
# locate it based upon its execution path.
#
# NOTE: It is recommend that this variable not be set here but in
# /etc/profile.d or equivalent. Some options (such as
# --config) may react strangely otherwise.
#
# export HADOOP_CONF_DIR=${HADOOP_HOME}/etc/hadoop
# The maximum amount of heap to use (Java -Xmx). If no unit
# is provided, it will be converted to MB. Daemons will
# prefer any Xmx setting in their respective _OPT variable.
# There is no default; the JVM will autoscale based upon machine
# memory size.
# export HADOOP_HEAPSIZE_MAX=
# The minimum amount of heap to use (Java -Xms). If no unit
# is provided, it will be converted to MB. Daemons will
# prefer any Xms setting in their respective _OPT variable.
# There is no default; the JVM will autoscale based upon machine
# memory size.
# export HADOOP_HEAPSIZE_MIN=
# Enable extra debugging of Hadoop's JAAS binding, used to set up
# Kerberos security.
# export HADOOP_JAAS_DEBUG=true
# Extra Java runtime options for all Hadoop commands. We don't support
# IPv6 yet/still, so by default the preference is set to IPv4.
# export HADOOP_OPTS="-Djava.net.preferIPv4Stack=true"
# For Kerberos debugging, an extended option set logs more information
# export HADOOP_OPTS="-Djava.net.preferIPv4Stack=true -Dsun.security.krb5.debug=true -Dsun.security.spnego.debug"
# Some parts of the shell code may do special things dependent upon
# the operating system. We have to set this here. See the next
# section as to why....
export HADOOP_OS_TYPE=${HADOOP_OS_TYPE:-$(uname -s)}
# Extra Java runtime options for some Hadoop commands
# and clients (i.e., hdfs dfs -blah). These get appended to HADOOP_OPTS for
# such commands. In most cases, # this should be left empty and
# let users supply it on the command line.
# export HADOOP_CLIENT_OPTS=""
#
# A note about classpaths.
#
# By default, Apache Hadoop overrides Java's CLASSPATH
# environment variable. It is configured such
# that it starts out blank with new entries added after passing
# a series of checks (file/dir exists, not already listed aka
# de-deduplication). During de-deduplication, wildcards and/or
# directories are *NOT* expanded to keep it simple. Therefore,
# if the computed classpath has two specific mentions of
# awesome-methods-1.0.jar, only the first one added will be seen.
# If two directories are in the classpath that both contain
# awesome-methods-1.0.jar, then Java will pick up both versions.
# An additional, custom CLASSPATH. Site-wide configs should be
# handled via the shellprofile functionality, utilizing the
# hadoop_add_classpath function for greater control and much
# harder for apps/end-users to accidentally override.
# Similarly, end users should utilize ${HOME}/.hadooprc .
# This variable should ideally only be used as a short-cut,
# interactive way for temporary additions on the command line.
# export HADOOP_CLASSPATH="/some/cool/path/on/your/machine"
# Should HADOOP_CLASSPATH be first in the official CLASSPATH?
# export HADOOP_USER_CLASSPATH_FIRST="yes"
# If HADOOP_USE_CLIENT_CLASSLOADER is set, the classpath along
# with the main jar are handled by a separate isolated
# client classloader when 'hadoop jar', 'yarn jar', or 'mapred job'
# is utilized. If it is set, HADOOP_CLASSPATH and
# HADOOP_USER_CLASSPATH_FIRST are ignored.
# export HADOOP_USE_CLIENT_CLASSLOADER=true
# HADOOP_CLIENT_CLASSLOADER_SYSTEM_CLASSES overrides the default definition of
# system classes for the client classloader when HADOOP_USE_CLIENT_CLASSLOADER
# is enabled. Names ending in '.' (period) are treated as package names, and
# names starting with a '-' are treated as negative matches. For example,
# export HADOOP_CLIENT_CLASSLOADER_SYSTEM_CLASSES="-org.apache.hadoop.UserClass,java.,javax.,org.apache.hadoop."
# Enable optional, bundled Hadoop features
# This is a comma delimited list. It may NOT be overridden via .hadooprc
# Entries may be added/removed as needed.
# export HADOOP_OPTIONAL_TOOLS="@@@HADOOP_OPTIONAL_TOOLS@@@"
###
# Options for remote shell connectivity
###
# There are some optional components of hadoop that allow for
# command and control of remote hosts. For example,
# start-dfs.sh will attempt to bring up all NNs, DNS, etc.
# Options to pass to SSH when one of the "log into a host and
# start/stop daemons" scripts is executed
# export HADOOP_SSH_OPTS="-o BatchMode=yes -o StrictHostKeyChecking=no -o ConnectTimeout=10s"
# The built-in ssh handler will limit itself to 10 simultaneous connections.
# For pdsh users, this sets the fanout size ( -f )
# Change this to increase/decrease as necessary.
# export HADOOP_SSH_PARALLEL=10
# Filename which contains all of the hosts for any remote execution
# helper scripts # such as workers.sh, start-dfs.sh, etc.
# export HADOOP_WORKERS="${HADOOP_CONF_DIR}/workers"
###
# Options for all daemons
###
#
#
# Many options may also be specified as Java properties. It is
# very common, and in many cases, desirable, to hard-set these
# in daemon _OPTS variables. Where applicable, the appropriate
# Java property is also identified. Note that many are re-used
# or set differently in certain contexts (e.g., secure vs
# non-secure)
#
# Where (primarily) daemon log files are stored.
# ${HADOOP_HOME}/logs by default.
# Java property: hadoop.log.dir
# export HADOOP_LOG_DIR=${HADOOP_HOME}/logs
# A string representing this instance of hadoop. $USER by default.
# This is used in writing log and pid files, so keep that in mind!
# Java property: hadoop.id.str
# export HADOOP_IDENT_STRING=$USER
# How many seconds to pause after stopping a daemon
# export HADOOP_STOP_TIMEOUT=5
# Where pid files are stored. /tmp by default.
# export HADOOP_PID_DIR=/tmp
# Default log4j setting for interactive commands
# Java property: hadoop.root.logger
# export HADOOP_ROOT_LOGGER=INFO,console
# Default log4j setting for daemons spawned explicitly by
# --daemon option of hadoop, hdfs, mapred and yarn command.
# Java property: hadoop.root.logger
# export HADOOP_DAEMON_ROOT_LOGGER=INFO,RFA
# Default log level and output location for security-related messages.
# You will almost certainly want to change this on a per-daemon basis via
# the Java property (i.e., -Dhadoop.security.logger=foo). (Note that the
# defaults for the NN and 2NN override this by default.)
# Java property: hadoop.security.logger
# export HADOOP_SECURITY_LOGGER=INFO,NullAppender
# Default process priority level
# Note that sub-processes will also run at this level!
# export HADOOP_NICENESS=0
# Default name for the service level authorization file
# Java property: hadoop.policy.file
# export HADOOP_POLICYFILE="hadoop-policy.xml"
#
# NOTE: this is not used by default! <-----
# You can define variables right here and then re-use them later on.
# For example, it is common to use the same garbage collection settings
# for all the daemons. So one could define:
#
# export HADOOP_GC_SETTINGS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps"
#
# .. and then use it as per the b option under the namenode.
###
# Secure/privileged execution
###
#
# Out of the box, Hadoop uses jsvc from Apache Commons to launch daemons
# on privileged ports. This functionality can be replaced by providing
# custom functions. See hadoop-functions.sh for more information.
#
# The jsvc implementation to use. Jsvc is required to run secure datanodes
# that bind to privileged ports to provide authentication of data transfer
# protocol. Jsvc is not required if SASL is configured for authentication of
# data transfer protocol using non-privileged ports.
# export JSVC_HOME=/usr/bin
#
# This directory contains pids for secure and privileged processes.
#export HADOOP_SECURE_PID_DIR=${HADOOP_PID_DIR}
#
# This directory contains the logs for secure and privileged processes.
# Java property: hadoop.log.dir
# export HADOOP_SECURE_LOG=${HADOOP_LOG_DIR}
#
# When running a secure daemon, the default value of HADOOP_IDENT_STRING
# ends up being a bit bogus. Therefore, by default, the code will
# replace HADOOP_IDENT_STRING with HADOOP_xx_SECURE_USER. If one wants
# to keep HADOOP_IDENT_STRING untouched, then uncomment this line.
# export HADOOP_SECURE_IDENT_PRESERVE="true"
###
# NameNode specific parameters
###
# Default log level and output location for file system related change
# messages. For non-namenode daemons, the Java property must be set in
# the appropriate _OPTS if one wants something other than INFO,NullAppender
# Java property: hdfs.audit.logger
# export HDFS_AUDIT_LOGGER=INFO,NullAppender
# Specify the JVM options to be used when starting the NameNode.
# These options will be appended to the options specified as HADOOP_OPTS
# and therefore may override any similar flags set in HADOOP_OPTS
#
# a) Set JMX options
# export HDFS_NAMENODE_OPTS="-Dcom.sun.management.jmxremote=true -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.port=1026"
#
# b) Set garbage collection logs
# export HDFS_NAMENODE_OPTS="${HADOOP_GC_SETTINGS} -Xloggc:${HADOOP_LOG_DIR}/gc-rm.log-$(date +'%Y%m%d%H%M')"
#
# c) ... or set them directly
# export HDFS_NAMENODE_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Xloggc:${HADOOP_LOG_DIR}/gc-rm.log-$(date +'%Y%m%d%H%M')"
# this is the default:
# export HDFS_NAMENODE_OPTS="-Dhadoop.security.logger=INFO,RFAS"
###
# SecondaryNameNode specific parameters
###
# Specify the JVM options to be used when starting the SecondaryNameNode.
# These options will be appended to the options specified as HADOOP_OPTS
# and therefore may override any similar flags set in HADOOP_OPTS
#
# This is the default:
# export HDFS_SECONDARYNAMENODE_OPTS="-Dhadoop.security.logger=INFO,RFAS"
###
# DataNode specific parameters
###
# Specify the JVM options to be used when starting the DataNode.
# These options will be appended to the options specified as HADOOP_OPTS
# and therefore may override any similar flags set in HADOOP_OPTS
#
# This is the default:
# export HDFS_DATANODE_OPTS="-Dhadoop.security.logger=ERROR,RFAS"
# On secure datanodes, user to run the datanode as after dropping privileges.
# This **MUST** be uncommented to enable secure HDFS if using privileged ports
# to provide authentication of data transfer protocol. This **MUST NOT** be
# defined if SASL is configured for authentication of data transfer protocol
# using non-privileged ports.
# This will replace the hadoop.id.str Java property in secure mode.
# export HDFS_DATANODE_SECURE_USER=hdfs
# Supplemental options for secure datanodes
# By default, Hadoop uses jsvc which needs to know to launch a
# server jvm.
# export HDFS_DATANODE_SECURE_EXTRA_OPTS="-jvm server"
###
# NFS3 Gateway specific parameters
###
# Specify the JVM options to be used when starting the NFS3 Gateway.
# These options will be appended to the options specified as HADOOP_OPTS
# and therefore may override any similar flags set in HADOOP_OPTS
#
# export HDFS_NFS3_OPTS=""
# Specify the JVM options to be used when starting the Hadoop portmapper.
# These options will be appended to the options specified as HADOOP_OPTS
# and therefore may override any similar flags set in HADOOP_OPTS
#
# export HDFS_PORTMAP_OPTS="-Xmx512m"
# Supplemental options for priviliged gateways
# By default, Hadoop uses jsvc which needs to know to launch a
# server jvm.
# export HDFS_NFS3_SECURE_EXTRA_OPTS="-jvm server"
# On privileged gateways, user to run the gateway as after dropping privileges
# This will replace the hadoop.id.str Java property in secure mode.
# export HDFS_NFS3_SECURE_USER=nfsserver
###
# ZKFailoverController specific parameters
###
# Specify the JVM options to be used when starting the ZKFailoverController.
# These options will be appended to the options specified as HADOOP_OPTS
# and therefore may override any similar flags set in HADOOP_OPTS
#
# export HDFS_ZKFC_OPTS=""
###
# QuorumJournalNode specific parameters
###
# Specify the JVM options to be used when starting the QuorumJournalNode.
# These options will be appended to the options specified as HADOOP_OPTS
# and therefore may override any similar flags set in HADOOP_OPTS
#
# export HDFS_JOURNALNODE_OPTS=""
###
# HDFS Balancer specific parameters
###
# Specify the JVM options to be used when starting the HDFS Balancer.
# These options will be appended to the options specified as HADOOP_OPTS
# and therefore may override any similar flags set in HADOOP_OPTS
#
# export HDFS_BALANCER_OPTS=""
###
# HDFS Mover specific parameters
###
# Specify the JVM options to be used when starting the HDFS Mover.
# These options will be appended to the options specified as HADOOP_OPTS
# and therefore may override any similar flags set in HADOOP_OPTS
#
# export HDFS_MOVER_OPTS=""
###
# Router-based HDFS Federation specific parameters
# Specify the JVM options to be used when starting the RBF Routers.
# These options will be appended to the options specified as HADOOP_OPTS
# and therefore may override any similar flags set in HADOOP_OPTS
#
# export HDFS_DFSROUTER_OPTS=""
###
# Ozone Manager specific parameters
###
# Specify the JVM options to be used when starting the Ozone Manager.
# These options will be appended to the options specified as HADOOP_OPTS
# and therefore may override any similar flags set in HADOOP_OPTS
#
# export HDFS_OM_OPTS=""
###
# HDFS StorageContainerManager specific parameters
###
# Specify the JVM options to be used when starting the HDFS Storage Container Manager.
# These options will be appended to the options specified as HADOOP_OPTS
# and therefore may override any similar flags set in HADOOP_OPTS
#
# export HDFS_STORAGECONTAINERMANAGER_OPTS=""
###
# Advanced Users Only!
###
#
# When building Hadoop, one can add the class paths to the commands
# via this special env var:
# export HADOOP_ENABLE_BUILD_PATHS="true"
#
# To prevent accidents, shell commands be (superficially) locked
# to only allow certain users to execute certain subcommands.
# It uses the format of (command)_(subcommand)_USER.
#
# For example, to limit who can execute the namenode command,
# export HDFS_NAMENODE_USER=hdfs
###
# Registry DNS specific parameters
###
# For privileged registry DNS, user to run as after dropping privileges
# This will replace the hadoop.id.str Java property in secure mode.
# export HADOOP_REGISTRYDNS_SECURE_USER=yarn
# Supplemental options for privileged registry DNS
# By default, Hadoop uses jsvc which needs to know to launch a
# server jvm.
# export HADOOP_REGISTRYDNS_SECURE_EXTRA_OPTS="-jvm server"

99
hadoop-hdds/common/src/main/conf/hadoop-metrics2.properties
View File

@ -1,99 +0,0 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# syntax: [prefix].[source|sink].[instance].[options]
# See javadoc of package-info.java for org.apache.hadoop.metrics2 for details
*.sink.file.class=org.apache.hadoop.metrics2.sink.FileSink
# default sampling period, in seconds
*.period=10
# The namenode-metrics.out will contain metrics from all context
#namenode.sink.file.filename=namenode-metrics.out
# Specifying a special sampling period for namenode:
#namenode.sink.*.period=8
#datanode.sink.file.filename=datanode-metrics.out
#resourcemanager.sink.file.filename=resourcemanager-metrics.out
#nodemanager.sink.file.filename=nodemanager-metrics.out
#mrappmaster.sink.file.filename=mrappmaster-metrics.out
#jobhistoryserver.sink.file.filename=jobhistoryserver-metrics.out
# the following example split metrics of different
# context to different sinks (in this case files)
#nodemanager.sink.file_jvm.class=org.apache.hadoop.metrics2.sink.FileSink
#nodemanager.sink.file_jvm.context=jvm
#nodemanager.sink.file_jvm.filename=nodemanager-jvm-metrics.out
#nodemanager.sink.file_mapred.class=org.apache.hadoop.metrics2.sink.FileSink
#nodemanager.sink.file_mapred.context=mapred
#nodemanager.sink.file_mapred.filename=nodemanager-mapred-metrics.out
#
# Below are for sending metrics to Ganglia
#
# for Ganglia 3.0 support
# *.sink.ganglia.class=org.apache.hadoop.metrics2.sink.ganglia.GangliaSink30
#
# for Ganglia 3.1 support
# *.sink.ganglia.class=org.apache.hadoop.metrics2.sink.ganglia.GangliaSink31
# *.sink.ganglia.period=10
# default for supportsparse is false
# *.sink.ganglia.supportsparse=true
#*.sink.ganglia.slope=jvm.metrics.gcCount=zero,jvm.metrics.memHeapUsedM=both
#*.sink.ganglia.dmax=jvm.metrics.threadsBlocked=70,jvm.metrics.memHeapUsedM=40
# Tag values to use for the ganglia prefix. If not defined no tags are used.
# If '*' all tags are used. If specifying multiple tags separate them with
# commas. Note that the last segment of the property name is the context name.
#
# A typical use of tags is separating the metrics by the HDFS rpc port
# and HDFS service rpc port.
# For example:
# With following HDFS configuration:
# dfs.namenode.rpc-address is set as namenodeAddress:9110
# dfs.namenode.servicerpc-address is set as namenodeAddress:9111
# If no tags are used, following metric would be gathered:
# rpc.rpc.NumOpenConnections
# If using "*.sink.ganglia.tagsForPrefix.rpc=port",
# following metrics would be gathered:
# rpc.rpc.port=9110.NumOpenConnections
# rpc.rpc.port=9111.NumOpenConnections
#
#*.sink.ganglia.tagsForPrefix.jvm=ProcessName
#*.sink.ganglia.tagsForPrefix.dfs=HAState,IsOutOfSync
#*.sink.ganglia.tagsForPrefix.rpc=port
#*.sink.ganglia.tagsForPrefix.rpcdetailed=port
#*.sink.ganglia.tagsForPrefix.metricssystem=*
#*.sink.ganglia.tagsForPrefix.ugi=*
#*.sink.ganglia.tagsForPrefix.mapred=
#namenode.sink.ganglia.servers=yourgangliahost_1:8649,yourgangliahost_2:8649
#datanode.sink.ganglia.servers=yourgangliahost_1:8649,yourgangliahost_2:8649
#resourcemanager.sink.ganglia.servers=yourgangliahost_1:8649,yourgangliahost_2:8649
#nodemanager.sink.ganglia.servers=yourgangliahost_1:8649,yourgangliahost_2:8649
#mrappmaster.sink.ganglia.servers=yourgangliahost_1:8649,yourgangliahost_2:8649
#jobhistoryserver.sink.ganglia.servers=yourgangliahost_1:8649,yourgangliahost_2:8649

275
hadoop-hdds/common/src/main/conf/hadoop-policy.xml
View File

@ -1,275 +0,0 @@
<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<!-- Put site-specific property overrides in this file. -->
<configuration>
<property>
<name>security.client.protocol.acl</name>
<value>*</value>
<description>ACL for ClientProtocol, which is used by user code
via the DistributedFileSystem.
The ACL is a comma-separated list of user and group names. The user and
group list is separated by a blank. For e.g. "alice,bob users,wheel".
A special value of "*" means all users are allowed.</description>
</property>
<property>
<name>security.client.datanode.protocol.acl</name>
<value>*</value>
<description>ACL for ClientDatanodeProtocol, the client-to-datanode protocol
for block recovery.
The ACL is a comma-separated list of user and group names. The user and
group list is separated by a blank. For e.g. "alice,bob users,wheel".
A special value of "*" means all users are allowed.</description>
</property>
<property>
<name>security.datanode.protocol.acl</name>
<value>*</value>
<description>ACL for DatanodeProtocol, which is used by datanodes to
communicate with the namenode.
The ACL is a comma-separated list of user and group names. The user and
group list is separated by a blank. For e.g. "alice,bob users,wheel".
A special value of "*" means all users are allowed.</description>
</property>
<property>
<name>security.inter.datanode.protocol.acl</name>
<value>*</value>
<description>ACL for InterDatanodeProtocol, the inter-datanode protocol
for updating generation timestamp.
The ACL is a comma-separated list of user and group names. The user and
group list is separated by a blank. For e.g. "alice,bob users,wheel".
A special value of "*" means all users are allowed.</description>
</property>
<property>
<name>security.namenode.protocol.acl</name>
<value>*</value>
<description>ACL for NamenodeProtocol, the protocol used by the secondary
namenode to communicate with the namenode.
The ACL is a comma-separated list of user and group names. The user and
group list is separated by a blank. For e.g. "alice,bob users,wheel".
A special value of "*" means all users are allowed.</description>
</property>
<property>
<name>security.admin.operations.protocol.acl</name>
<value>*</value>
<description>ACL for AdminOperationsProtocol. Used for admin commands.
The ACL is a comma-separated list of user and group names. The user and
group list is separated by a blank. For e.g. "alice,bob users,wheel".
A special value of "*" means all users are allowed.</description>
</property>
<property>
<name>security.refresh.user.mappings.protocol.acl</name>
<value>*</value>
<description>ACL for RefreshUserMappingsProtocol. Used to refresh
users mappings. The ACL is a comma-separated list of user and
group names. The user and group list is separated by a blank. For
e.g. "alice,bob users,wheel". A special value of "*" means all
users are allowed.</description>
</property>
<property>
<name>security.refresh.policy.protocol.acl</name>
<value>*</value>
<description>ACL for RefreshAuthorizationPolicyProtocol, used by the
dfsadmin and mradmin commands to refresh the security policy in-effect.
The ACL is a comma-separated list of user and group names. The user and
group list is separated by a blank. For e.g. "alice,bob users,wheel".
A special value of "*" means all users are allowed.</description>
</property>
<property>
<name>security.ha.service.protocol.acl</name>
<value>*</value>
<description>ACL for HAService protocol used by HAAdmin to manage the
active and stand-by states of namenode.</description>
</property>
<property>
<name>security.router.admin.protocol.acl</name>
<value>*</value>
<description>ACL for RouterAdmin Protocol. The ACL is a comma-separated
list of user and group names. The user and
group list is separated by a blank. For e.g. "alice,bob users,wheel".
A special value of "*" means all users are allowed.
</description>
</property>
<property>
<name>security.zkfc.protocol.acl</name>
<value>*</value>
<description>ACL for access to the ZK Failover Controller
</description>
</property>
<property>
<name>security.qjournal.service.protocol.acl</name>
<value>*</value>
<description>ACL for QJournalProtocol, used by the NN to communicate with
JNs when using the QuorumJournalManager for edit logs.</description>
</property>
<property>
<name>security.interqjournal.service.protocol.acl</name>
<value>*</value>
<description>ACL for InterQJournalProtocol, used by the JN to
communicate with other JN
</description>
</property>
<property>
<name>security.mrhs.client.protocol.acl</name>
<value>*</value>
<description>ACL for HSClientProtocol, used by job clients to
communciate with the MR History Server job status etc.
The ACL is a comma-separated list of user and group names. The user and
group list is separated by a blank. For e.g. "alice,bob users,wheel".
A special value of "*" means all users are allowed.</description>
</property>
<!-- YARN Protocols -->
<property>
<name>security.resourcetracker.protocol.acl</name>
<value>*</value>
<description>ACL for ResourceTrackerProtocol, used by the
ResourceManager and NodeManager to communicate with each other.
The ACL is a comma-separated list of user and group names. The user and
group list is separated by a blank. For e.g. "alice,bob users,wheel".
A special value of "*" means all users are allowed.</description>
</property>
<property>
<name>security.resourcemanager-administration.protocol.acl</name>
<value>*</value>
<description>ACL for ResourceManagerAdministrationProtocol, for admin commands.
The ACL is a comma-separated list of user and group names. The user and
group list is separated by a blank. For e.g. "alice,bob users,wheel".
A special value of "*" means all users are allowed.</description>
</property>
<property>
<name>security.applicationclient.protocol.acl</name>
<value>*</value>
<description>ACL for ApplicationClientProtocol, used by the ResourceManager
and applications submission clients to communicate with each other.
The ACL is a comma-separated list of user and group names. The user and
group list is separated by a blank. For e.g. "alice,bob users,wheel".
A special value of "*" means all users are allowed.</description>
</property>
<property>
<name>security.applicationmaster.protocol.acl</name>
<value>*</value>
<description>ACL for ApplicationMasterProtocol, used by the ResourceManager
and ApplicationMasters to communicate with each other.
The ACL is a comma-separated list of user and group names. The user and
group list is separated by a blank. For e.g. "alice,bob users,wheel".
A special value of "*" means all users are allowed.</description>
</property>
<property>
<name>security.containermanagement.protocol.acl</name>
<value>*</value>
<description>ACL for ContainerManagementProtocol protocol, used by the NodeManager
and ApplicationMasters to communicate with each other.
The ACL is a comma-separated list of user and group names. The user and
group list is separated by a blank. For e.g. "alice,bob users,wheel".
A special value of "*" means all users are allowed.</description>
</property>
<property>
<name>security.resourcelocalizer.protocol.acl</name>
<value>*</value>
<description>ACL for ResourceLocalizer protocol, used by the NodeManager
and ResourceLocalizer to communicate with each other.
The ACL is a comma-separated list of user and group names. The user and
group list is separated by a blank. For e.g. "alice,bob users,wheel".
A special value of "*" means all users are allowed.</description>
</property>
<property>
<name>security.job.task.protocol.acl</name>
<value>*</value>
<description>ACL for TaskUmbilicalProtocol, used by the map and reduce
tasks to communicate with the parent tasktracker.
The ACL is a comma-separated list of user and group names. The user and
group list is separated by a blank. For e.g. "alice,bob users,wheel".
A special value of "*" means all users are allowed.</description>
</property>
<property>
<name>security.job.client.protocol.acl</name>
<value>*</value>
<description>ACL for MRClientProtocol, used by job clients to
communciate with the MR ApplicationMaster to query job status etc.
The ACL is a comma-separated list of user and group names. The user and
group list is separated by a blank. For e.g. "alice,bob users,wheel".
A special value of "*" means all users are allowed.</description>
</property>
<property>
<name>security.applicationhistory.protocol.acl</name>
<value>*</value>
<description>ACL for ApplicationHistoryProtocol, used by the timeline
server and the generic history service client to communicate with each other.
The ACL is a comma-separated list of user and group names. The user and
group list is separated by a blank. For e.g. "alice,bob users,wheel".
A special value of "*" means all users are allowed.</description>
</property>
<property>
<name>security.collector-nodemanager.protocol.acl</name>
<value>*</value>
<description>ACL for CollectorNodemanagerProtocol, used by nodemanager
if timeline service v2 is enabled, for the timeline collector and nodemanager
to communicate with each other.
The ACL is a comma-separated list of user and group names. The user and
group list is separated by a blank. For e.g. "alice,bob users,wheel".
A special value of "*" means all users are allowed.</description>
</property>
<property>
<name>security.applicationmaster-nodemanager.applicationmaster.protocol.acl</name>
<value>*</value>
<description>ACL for ApplicationMasterProtocol, used by the Nodemanager
and ApplicationMasters to communicate.
The ACL is a comma-separated list of user and group names. The user and
group list is separated by a blank. For e.g. "alice,bob users,wheel".
A special value of "*" means all users are allowed.</description>
</property>
<property>
<name>security.distributedscheduling.protocol.acl</name>
<value>*</value>
<description>ACL for DistributedSchedulingAMProtocol, used by the Nodemanager
and Resourcemanager to communicate.
The ACL is a comma-separated list of user and group names. The user and
group list is separated by a blank. For e.g. "alice,bob users,wheel".
A special value of "*" means all users are allowed.</description>
</property>
</configuration>

252
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
View File

@ -1,252 +0,0 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations under
* the License.
*/
package org.apache.hadoop.hdds;
import org.apache.hadoop.hdds.utils.db.DBProfile;
/**
* This class contains constants for configuration keys and default values
* used in hdds.
*/
public final class HddsConfigKeys {
public static final String HDDS_HEARTBEAT_INTERVAL =
"hdds.heartbeat.interval";
public static final String HDDS_HEARTBEAT_INTERVAL_DEFAULT =
"30s";
public static final String HDDS_NODE_REPORT_INTERVAL =
"hdds.node.report.interval";
public static final String HDDS_NODE_REPORT_INTERVAL_DEFAULT =
"60s";
public static final String HDDS_CONTAINER_REPORT_INTERVAL =
"hdds.container.report.interval";
public static final String HDDS_CONTAINER_REPORT_INTERVAL_DEFAULT =
"60s";
public static final String HDDS_PIPELINE_REPORT_INTERVAL =
"hdds.pipeline.report.interval";
public static final String HDDS_PIPELINE_REPORT_INTERVAL_DEFAULT =
"60s";
public static final String HDDS_COMMAND_STATUS_REPORT_INTERVAL =
"hdds.command.status.report.interval";
public static final String HDDS_COMMAND_STATUS_REPORT_INTERVAL_DEFAULT =
"60s";
public static final String HDDS_CONTAINER_ACTION_MAX_LIMIT =
"hdds.container.action.max.limit";
public static final int HDDS_CONTAINER_ACTION_MAX_LIMIT_DEFAULT =
20;
public static final String HDDS_PIPELINE_ACTION_MAX_LIMIT =
"hdds.pipeline.action.max.limit";
public static final int HDDS_PIPELINE_ACTION_MAX_LIMIT_DEFAULT =
20;
// Configuration to allow volume choosing policy.
public static final String HDDS_DATANODE_VOLUME_CHOOSING_POLICY =
"hdds.datanode.volume.choosing.policy";
// DB PKIProfile used by ROCKDB instances.
public static final String HDDS_DB_PROFILE = "hdds.db.profile";
public static final DBProfile HDDS_DEFAULT_DB_PROFILE = DBProfile.DISK;
// Once a container usage crosses this threshold, it is eligible for
// closing.
public static final String HDDS_CONTAINER_CLOSE_THRESHOLD =
"hdds.container.close.threshold";
public static final float HDDS_CONTAINER_CLOSE_THRESHOLD_DEFAULT = 0.9f;
public static final String HDDS_SCM_SAFEMODE_ENABLED =
"hdds.scm.safemode.enabled";
public static final boolean HDDS_SCM_SAFEMODE_ENABLED_DEFAULT = true;
public static final String HDDS_SCM_SAFEMODE_MIN_DATANODE =
"hdds.scm.safemode.min.datanode";
public static final int HDDS_SCM_SAFEMODE_MIN_DATANODE_DEFAULT = 1;
public static final String
HDDS_SCM_WAIT_TIME_AFTER_SAFE_MODE_EXIT =
"hdds.scm.wait.time.after.safemode.exit";
public static final String
HDDS_SCM_WAIT_TIME_AFTER_SAFE_MODE_EXIT_DEFAULT = "5m";
public static final String HDDS_SCM_SAFEMODE_PIPELINE_AVAILABILITY_CHECK =
"hdds.scm.safemode.pipeline-availability.check";
public static final boolean
HDDS_SCM_SAFEMODE_PIPELINE_AVAILABILITY_CHECK_DEFAULT = false;
// % of containers which should have at least one reported replica
// before SCM comes out of safe mode.
public static final String HDDS_SCM_SAFEMODE_THRESHOLD_PCT =
"hdds.scm.safemode.threshold.pct";
public static final double HDDS_SCM_SAFEMODE_THRESHOLD_PCT_DEFAULT = 0.99;
// percentage of healthy pipelines, where all 3 datanodes are reported in the
// pipeline.
public static final String HDDS_SCM_SAFEMODE_HEALTHY_PIPELINE_THRESHOLD_PCT =
"hdds.scm.safemode.healthy.pipelie.pct";
public static final double
HDDS_SCM_SAFEMODE_HEALTHY_PIPELINE_THRESHOLD_PCT_DEFAULT = 0.10;
public static final String HDDS_SCM_SAFEMODE_ONE_NODE_REPORTED_PIPELINE_PCT =
"hdds.scm.safemode.atleast.one.node.reported.pipeline.pct";
public static final double
HDDS_SCM_SAFEMODE_ONE_NODE_REPORTED_PIPELINE_PCT_DEFAULT = 0.90;
public static final String HDDS_LOCK_MAX_CONCURRENCY =
"hdds.lock.max.concurrency";
public static final int HDDS_LOCK_MAX_CONCURRENCY_DEFAULT = 100;
// This configuration setting is used as a fallback location by all
// Ozone/HDDS services for their metadata. It is useful as a single
// config point for test/PoC clusters.
//
// In any real cluster where performance matters, the SCM, OM and DN
// metadata locations must be configured explicitly.
public static final String OZONE_METADATA_DIRS = "ozone.metadata.dirs";
public static final String HDDS_PROMETHEUS_ENABLED =
"hdds.prometheus.endpoint.enabled";
public static final String HDDS_PROFILER_ENABLED =
"hdds.profiler.endpoint.enabled";
public static final String HDDS_KEY_LEN = "hdds.key.len";
public static final int HDDS_DEFAULT_KEY_LEN = 2048;
public static final String HDDS_KEY_ALGORITHM = "hdds.key.algo";
public static final String HDDS_DEFAULT_KEY_ALGORITHM = "RSA";
public static final String HDDS_SECURITY_PROVIDER = "hdds.security.provider";
public static final String HDDS_DEFAULT_SECURITY_PROVIDER = "BC";
public static final String HDDS_KEY_DIR_NAME = "hdds.key.dir.name";
public static final String HDDS_KEY_DIR_NAME_DEFAULT = "keys";
// TODO : Talk to StorageIO classes and see if they can return a secure
// storage location for each node.
public static final String HDDS_METADATA_DIR_NAME = "hdds.metadata.dir";
public static final String HDDS_PRIVATE_KEY_FILE_NAME =
"hdds.priv.key.file.name";
public static final String HDDS_PRIVATE_KEY_FILE_NAME_DEFAULT = "private.pem";
public static final String HDDS_PUBLIC_KEY_FILE_NAME = "hdds.public.key.file"
+ ".name";
public static final String HDDS_PUBLIC_KEY_FILE_NAME_DEFAULT = "public.pem";
public static final String HDDS_BLOCK_TOKEN_EXPIRY_TIME =
"hdds.block.token.expiry.time";
public static final String HDDS_BLOCK_TOKEN_EXPIRY_TIME_DEFAULT = "1d";
/**
* Maximum duration of certificates issued by SCM including Self-Signed Roots.
* The formats accepted are based on the ISO-8601 duration format PnDTnHnMn.nS
* Default value is 5 years and written as P1865D.
*/
public static final String HDDS_X509_MAX_DURATION = "hdds.x509.max.duration";
// Limit Certificate duration to a max value of 5 years.
public static final String HDDS_X509_MAX_DURATION_DEFAULT= "P1865D";
public static final String HDDS_X509_SIGNATURE_ALGO =
"hdds.x509.signature.algorithm";
public static final String HDDS_X509_SIGNATURE_ALGO_DEFAULT = "SHA256withRSA";
public static final String HDDS_BLOCK_TOKEN_ENABLED =
"hdds.block.token.enabled";
public static final boolean HDDS_BLOCK_TOKEN_ENABLED_DEFAULT = false;
public static final String HDDS_X509_DIR_NAME = "hdds.x509.dir.name";
public static final String HDDS_X509_DIR_NAME_DEFAULT = "certs";
public static final String HDDS_X509_FILE_NAME = "hdds.x509.file.name";
public static final String HDDS_X509_FILE_NAME_DEFAULT = "certificate.crt";
/**
* Default duration of certificates issued by SCM CA.
* The formats accepted are based on the ISO-8601 duration format PnDTnHnMn.nS
* Default value is 5 years and written as P1865D.
*/
public static final String HDDS_X509_DEFAULT_DURATION = "hdds.x509.default" +
".duration";
// Default Certificate duration to one year.
public static final String HDDS_X509_DEFAULT_DURATION_DEFAULT = "P365D";
/**
* Do not instantiate.
*/
private HddsConfigKeys() {
}
// Enable TLS for GRPC clients/server in ozone.
public static final String HDDS_GRPC_TLS_ENABLED = "hdds.grpc.tls.enabled";
public static final boolean HDDS_GRPC_TLS_ENABLED_DEFAULT = false;
// Choose TLS provider the default is set to OPENSSL for better performance.
public static final String HDDS_GRPC_TLS_PROVIDER = "hdds.grpc.tls.provider";
public static final String HDDS_GRPC_TLS_PROVIDER_DEFAULT = "OPENSSL";
// Test only settings for using test signed certificate, authority assume to
// be localhost.
public static final String HDDS_GRPC_TLS_TEST_CERT = "hdds.grpc.tls" +
".test.cert";
public static final boolean HDDS_GRPC_TLS_TEST_CERT_DEFAULT = false;
// Comma separated acls (users, groups) allowing clients accessing
// datanode container protocol
// when hadoop.security.authorization is true, this needs to be set in
// hadoop-policy.xml, "*" allows all users/groups to access.
public static final String
HDDS_SECURITY_CLIENT_DATANODE_CONTAINER_PROTOCOL_ACL =
"hdds.security.client.datanode.container.protocol.acl";
// Comma separated acls (users, groups) allowing clients accessing
// scm container protocol
// when hadoop.security.authorization is true, this needs to be set in
// hadoop-policy.xml, "*" allows all users/groups to access.
public static final String HDDS_SECURITY_CLIENT_SCM_CONTAINER_PROTOCOL_ACL =
"hdds.security.client.scm.container.protocol.acl";
// Comma separated acls (users, groups) allowing clients accessing
// scm block protocol
// when hadoop.security.authorization is true, this needs to be set in
// hadoop-policy.xml, "*" allows all users/groups to access.
public static final String HDDS_SECURITY_CLIENT_SCM_BLOCK_PROTOCOL_ACL =
"hdds.security.client.scm.block.protocol.acl";
// Comma separated acls (users, groups) allowing clients accessing
// scm certificate protocol
// when hadoop.security.authorization is true, this needs to be set in
// hadoop-policy.xml, "*" allows all users/groups to access.
public static final String HDDS_SECURITY_CLIENT_SCM_CERTIFICATE_PROTOCOL_ACL =
"hdds.security.client.scm.certificate.protocol.acl";
// Determines if the Container Chunk Manager will write user data to disk
// Set to false only for specific performance tests
public static final String HDDS_CONTAINER_PERSISTDATA =
"hdds.container.chunk.persistdata";
public static final boolean HDDS_CONTAINER_PERSISTDATA_DEFAULT = true;
public static final String HDDS_CONTAINER_SCRUB_ENABLED =
"hdds.container.scrub.enabled";
public static final boolean HDDS_CONTAINER_SCRUB_ENABLED_DEFAULT = false;
public static final String HDDS_DATANODE_HTTP_ENABLED_KEY =
"hdds.datanode.http.enabled";
public static final String HDDS_DATANODE_HTTP_BIND_HOST_KEY =
"hdds.datanode.http-bind-host";
public static final String HDDS_DATANODE_HTTPS_BIND_HOST_KEY =
"hdds.datanode.https-bind-host";
public static final String HDDS_DATANODE_HTTP_ADDRESS_KEY =
"hdds.datanode.http-address";
public static final String HDDS_DATANODE_HTTPS_ADDRESS_KEY =
"hdds.datanode.https-address";
public static final String HDDS_DATANODE_HTTP_BIND_HOST_DEFAULT = "0.0.0.0";
public static final int HDDS_DATANODE_HTTP_BIND_PORT_DEFAULT = 9882;
public static final int HDDS_DATANODE_HTTPS_BIND_PORT_DEFAULT = 9883;
public static final String
HDDS_DATANODE_HTTP_KERBEROS_PRINCIPAL_KEY =
"hdds.datanode.http.kerberos.principal";
public static final String
HDDS_DATANODE_HTTP_KERBEROS_KEYTAB_FILE_KEY =
"hdds.datanode.http.kerberos.keytab";
}

53
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsIdFactory.java
View File

@ -1,53 +0,0 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdds;
import java.util.UUID;
import java.util.concurrent.atomic.AtomicLong;
/**
* HDDS Id generator.
*/
public final class HddsIdFactory {
private HddsIdFactory() {
}
private static final AtomicLong LONG_COUNTER = new AtomicLong(
System.currentTimeMillis());
/**
* Returns an incrementing long. This class doesn't
* persist initial value for long Id's, so incremental id's after restart
* may collide with previously generated Id's.
*
* @return long
*/
public static long getLongId() {
return LONG_COUNTER.incrementAndGet();
}
/**
* Returns a uuid.
*
* @return UUID.
*/
public static UUID getUUId() {
return UUID.randomUUID();
}
}

505
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java
View File

@ -1,505 +0,0 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdds;
import javax.management.ObjectName;
import java.io.IOException;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.net.InetSocketAddress;
import java.net.UnknownHostException;
import java.util.Calendar;
import java.util.Collection;
import java.util.HashSet;
import java.util.Map;
import java.util.Optional;
import java.util.TimeZone;
import java.util.concurrent.TimeUnit;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
import org.apache.hadoop.hdds.protocolPB.SCMSecurityProtocolClientSideTranslatorPB;
import org.apache.hadoop.hdds.protocolPB.SCMSecurityProtocolPB;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.SCMSecurityProtocol;
import org.apache.hadoop.hdds.scm.protocolPB.ScmBlockLocationProtocolPB;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.io.retry.RetryPolicies;
import org.apache.hadoop.io.retry.RetryPolicy;
import org.apache.hadoop.ipc.Client;
import org.apache.hadoop.ipc.ProtobufRpcEngine;
import org.apache.hadoop.ipc.RPC;
import org.apache.hadoop.metrics2.MetricsSystem;
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
import org.apache.hadoop.metrics2.source.JvmMetrics;
import org.apache.hadoop.metrics2.util.MBeans;
import org.apache.hadoop.net.DNS;
import org.apache.hadoop.net.NetUtils;
import com.google.common.net.HostAndPort;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_DNS_INTERFACE_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_DNS_NAMESERVER_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_HOST_NAME_KEY;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ENABLED;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ENABLED_DEFAULT;
import org.apache.hadoop.security.UserGroupInformation;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* HDDS specific stateless utility functions.
*/
@InterfaceAudience.Private
@InterfaceStability.Stable
public final class HddsUtils {
private static final Logger LOG = LoggerFactory.getLogger(HddsUtils.class);
/**
* The service ID of the solitary Ozone SCM service.
*/
public static final String OZONE_SCM_SERVICE_ID = "OzoneScmService";
public static final String OZONE_SCM_SERVICE_INSTANCE_ID =
"OzoneScmServiceInstance";
private static final TimeZone UTC_ZONE = TimeZone.getTimeZone("UTC");
private static final int NO_PORT = -1;
private HddsUtils() {
}
/**
* Retrieve the socket address that should be used by clients to connect
* to the SCM.
*
* @param conf
* @return Target InetSocketAddress for the SCM client endpoint.
*/
public static InetSocketAddress getScmAddressForClients(Configuration conf) {
Optional<String> host = getHostNameFromConfigKeys(conf,
ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY);
if (!host.isPresent()) {
// Fallback to Ozone SCM names.
Collection<InetSocketAddress> scmAddresses = getSCMAddresses(conf);
if (scmAddresses.size() > 1) {
throw new IllegalArgumentException(
ScmConfigKeys.OZONE_SCM_NAMES +
" must contain a single hostname. Multiple SCM hosts are " +
"currently unsupported");
}
host = Optional.of(scmAddresses.iterator().next().getHostName());
}
if (!host.isPresent()) {
throw new IllegalArgumentException(
ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY + " must be defined. See"
+ " https://wiki.apache.org/hadoop/Ozone#Configuration for "
+ "details"
+ " on configuring Ozone.");
}
final Optional<Integer> port = getPortNumberFromConfigKeys(conf,
ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY);
return NetUtils.createSocketAddr(host.get() + ":" + port
.orElse(ScmConfigKeys.OZONE_SCM_CLIENT_PORT_DEFAULT));
}
/**
* Retrieve the socket address that should be used by clients to connect
* to the SCM for block service. If
* {@link ScmConfigKeys#OZONE_SCM_BLOCK_CLIENT_ADDRESS_KEY} is not defined
* then {@link ScmConfigKeys#OZONE_SCM_CLIENT_ADDRESS_KEY} is used. If neither
* is defined then {@link ScmConfigKeys#OZONE_SCM_NAMES} is used.
*
* @param conf
* @return Target InetSocketAddress for the SCM block client endpoint.
* @throws IllegalArgumentException if configuration is not defined.
*/
public static InetSocketAddress getScmAddressForBlockClients(
Configuration conf) {
Optional<String> host = getHostNameFromConfigKeys(conf,
ScmConfigKeys.OZONE_SCM_BLOCK_CLIENT_ADDRESS_KEY);
if (!host.isPresent()) {
host = getHostNameFromConfigKeys(conf,
ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY);
}
if (!host.isPresent()) {
// Fallback to Ozone SCM names.
Collection<InetSocketAddress> scmAddresses = getSCMAddresses(conf);
if (scmAddresses.size() > 1) {
throw new IllegalArgumentException(
ScmConfigKeys.OZONE_SCM_NAMES +
" must contain a single hostname. Multiple SCM hosts are " +
"currently unsupported");
}
host = Optional.of(scmAddresses.iterator().next().getHostName());
}
if (!host.isPresent()) {
throw new IllegalArgumentException(
ScmConfigKeys.OZONE_SCM_BLOCK_CLIENT_ADDRESS_KEY
+ " must be defined. See"
+ " https://wiki.apache.org/hadoop/Ozone#Configuration"
+ " for details on configuring Ozone.");
}
final Optional<Integer> port = getPortNumberFromConfigKeys(conf,
ScmConfigKeys.OZONE_SCM_BLOCK_CLIENT_ADDRESS_KEY);
return NetUtils.createSocketAddr(host.get() + ":" + port
.orElse(ScmConfigKeys.OZONE_SCM_BLOCK_CLIENT_PORT_DEFAULT));
}
/**
* Create a scm security client.
* @param conf - Ozone configuration.
*
* @return {@link SCMSecurityProtocol}
* @throws IOException
*/
public static SCMSecurityProtocolClientSideTranslatorPB getScmSecurityClient(
OzoneConfiguration conf) throws IOException {
RPC.setProtocolEngine(conf, SCMSecurityProtocolPB.class,
ProtobufRpcEngine.class);
long scmVersion =
RPC.getProtocolVersion(ScmBlockLocationProtocolPB.class);
InetSocketAddress address =
getScmAddressForSecurityProtocol(conf);
RetryPolicy retryPolicy =
RetryPolicies.retryForeverWithFixedSleep(
1000, TimeUnit.MILLISECONDS);
SCMSecurityProtocolClientSideTranslatorPB scmSecurityClient =
new SCMSecurityProtocolClientSideTranslatorPB(
RPC.getProtocolProxy(SCMSecurityProtocolPB.class, scmVersion,
address, UserGroupInformation.getCurrentUser(),
conf, NetUtils.getDefaultSocketFactory(conf),
Client.getRpcTimeout(conf), retryPolicy).getProxy());
return scmSecurityClient;
}
/**
* Retrieve the hostname, trying the supplied config keys in order.
* Each config value may be absent, or if present in the format
* host:port (the :port part is optional).
*
* @param conf - Conf
* @param keys a list of configuration key names.
*
* @return first hostname component found from the given keys, or absent.
* @throws IllegalArgumentException if any values are not in the 'host'
* or host:port format.
*/
public static Optional<String> getHostNameFromConfigKeys(Configuration conf,
String... keys) {
for (final String key : keys) {
final String value = conf.getTrimmed(key);
final Optional<String> hostName = getHostName(value);
if (hostName.isPresent()) {
return hostName;
}
}
return Optional.empty();
}
/**
* Gets the hostname or Indicates that it is absent.
* @param value host or host:port
* @return hostname
*/
public static Optional<String> getHostName(String value) {
if ((value == null) || value.isEmpty()) {
return Optional.empty();
}
String hostname = value.replaceAll("\\:[0-9]+$", "");
if (hostname.length() == 0) {
return Optional.empty();
} else {
return Optional.of(hostname);
}
}
/**
* Gets the port if there is one, throws otherwise.
* @param value String in host:port format.
* @return Port
*/
public static Optional<Integer> getHostPort(String value) {
if ((value == null) || value.isEmpty()) {
return Optional.empty();
}
int port = HostAndPort.fromString(value).getPortOrDefault(NO_PORT);
if (port == NO_PORT) {
return Optional.empty();
} else {
return Optional.of(port);
}
}
/**
* Retrieve the port number, trying the supplied config keys in order.
* Each config value may be absent, or if present in the format
* host:port (the :port part is optional).
*
* @param conf Conf
* @param keys a list of configuration key names.
*
* @return first port number component found from the given keys, or absent.
* @throws IllegalArgumentException if any values are not in the 'host'
* or host:port format.
*/
public static Optional<Integer> getPortNumberFromConfigKeys(
Configuration conf, String... keys) {
for (final String key : keys) {
final String value = conf.getTrimmed(key);
final Optional<Integer> hostPort = getHostPort(value);
if (hostPort.isPresent()) {
return hostPort;
}
}
return Optional.empty();
}
/**
* Retrieve the socket addresses of all storage container managers.
*
* @param conf
* @return A collection of SCM addresses
* @throws IllegalArgumentException If the configuration is invalid
*/
public static Collection<InetSocketAddress> getSCMAddresses(
Configuration conf) throws IllegalArgumentException {
Collection<InetSocketAddress> addresses =
new HashSet<InetSocketAddress>();
Collection<String> names =
conf.getTrimmedStringCollection(ScmConfigKeys.OZONE_SCM_NAMES);
if (names == null || names.isEmpty()) {
throw new IllegalArgumentException(ScmConfigKeys.OZONE_SCM_NAMES
+ " need to be a set of valid DNS names or IP addresses."
+ " Null or empty address list found.");
}
final Optional<Integer> defaultPort = Optional
.of(ScmConfigKeys.OZONE_SCM_DEFAULT_PORT);
for (String address : names) {
Optional<String> hostname = getHostName(address);
if (!hostname.isPresent()) {
throw new IllegalArgumentException("Invalid hostname for SCM: "
+ hostname);
}
Optional<Integer> port = getHostPort(address);
InetSocketAddress addr = NetUtils.createSocketAddr(hostname.get(),
port.orElse(defaultPort.get()));
addresses.add(addr);
}
return addresses;
}
public static boolean isHddsEnabled(Configuration conf) {
return conf.getBoolean(OZONE_ENABLED, OZONE_ENABLED_DEFAULT);
}
/**
* Returns the hostname for this datanode. If the hostname is not
* explicitly configured in the given config, then it is determined
* via the DNS class.
*
* @param conf Configuration
*
* @return the hostname (NB: may not be a FQDN)
* @throws UnknownHostException if the dfs.datanode.dns.interface
* option is used and the hostname can not be determined
*/
public static String getHostName(Configuration conf)
throws UnknownHostException {
String name = conf.get(DFS_DATANODE_HOST_NAME_KEY);
if (name == null) {
String dnsInterface = conf.get(
CommonConfigurationKeys.HADOOP_SECURITY_DNS_INTERFACE_KEY);
String nameServer = conf.get(
CommonConfigurationKeys.HADOOP_SECURITY_DNS_NAMESERVER_KEY);
boolean fallbackToHosts = false;
if (dnsInterface == null) {
// Try the legacy configuration keys.
dnsInterface = conf.get(DFS_DATANODE_DNS_INTERFACE_KEY);
nameServer = conf.get(DFS_DATANODE_DNS_NAMESERVER_KEY);
} else {
// If HADOOP_SECURITY_DNS_* is set then also attempt hosts file
// resolution if DNS fails. We will not use hosts file resolution
// by default to avoid breaking existing clusters.
fallbackToHosts = true;
}
name = DNS.getDefaultHost(dnsInterface, nameServer, fallbackToHosts);
}
return name;
}
/**
* Checks if the container command is read only or not.
* @param proto ContainerCommand Request proto
* @return True if its readOnly , false otherwise.
*/
public static boolean isReadOnly(
ContainerProtos.ContainerCommandRequestProto proto) {
switch (proto.getCmdType()) {
case ReadContainer:
case ReadChunk:
case ListBlock:
case GetBlock:
case GetSmallFile:
case ListContainer:
case ListChunk:
case GetCommittedBlockLength:
return true;
case CloseContainer:
case WriteChunk:
case UpdateContainer:
case CompactChunk:
case CreateContainer:
case DeleteChunk:
case DeleteContainer:
case DeleteBlock:
case PutBlock:
case PutSmallFile:
default:
return false;
}
}
/**
* Register the provided MBean with additional JMX ObjectName properties.
* If additional properties are not supported then fallback to registering
* without properties.
*
* @param serviceName - see {@link MBeans#register}
* @param mBeanName - see {@link MBeans#register}
* @param jmxProperties - additional JMX ObjectName properties.
* @param mBean - the MBean to register.
* @return the named used to register the MBean.
*/
public static ObjectName registerWithJmxProperties(
String serviceName, String mBeanName, Map<String, String> jmxProperties,
Object mBean) {
try {
// Check support for registering with additional properties.
final Method registerMethod = MBeans.class.getMethod(
"register", String.class, String.class,
Map.class, Object.class);
return (ObjectName) registerMethod.invoke(
null, serviceName, mBeanName, jmxProperties, mBean);
} catch (NoSuchMethodException | IllegalAccessException |
InvocationTargetException e) {
// Fallback
if (LOG.isTraceEnabled()) {
LOG.trace("Registering MBean {} without additional properties {}",
mBeanName, jmxProperties);
}
return MBeans.register(serviceName, mBeanName, mBean);
}
}
/**
* Get the current UTC time in milliseconds.
* @return the current UTC time in milliseconds.
*/
public static long getUtcTime() {
return Calendar.getInstance(UTC_ZONE).getTimeInMillis();
}
/**
* Retrieve the socket address that should be used by clients to connect
* to the SCM for
* {@link org.apache.hadoop.hdds.protocol.SCMSecurityProtocol}. If
* {@link ScmConfigKeys#OZONE_SCM_SECURITY_SERVICE_ADDRESS_KEY} is not defined
* then {@link ScmConfigKeys#OZONE_SCM_CLIENT_ADDRESS_KEY} is used. If neither
* is defined then {@link ScmConfigKeys#OZONE_SCM_NAMES} is used.
*
* @param conf
* @return Target InetSocketAddress for the SCM block client endpoint.
* @throws IllegalArgumentException if configuration is not defined.
*/
public static InetSocketAddress getScmAddressForSecurityProtocol(
Configuration conf) {
Optional<String> host = getHostNameFromConfigKeys(conf,
ScmConfigKeys.OZONE_SCM_SECURITY_SERVICE_ADDRESS_KEY);
if (!host.isPresent()) {
host = getHostNameFromConfigKeys(conf,
ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY);
}
if (!host.isPresent()) {
// Fallback to Ozone SCM names.
Collection<InetSocketAddress> scmAddresses = getSCMAddresses(conf);
if (scmAddresses.size() > 1) {
throw new IllegalArgumentException(
ScmConfigKeys.OZONE_SCM_NAMES +
" must contain a single hostname. Multiple SCM hosts are " +
"currently unsupported");
}
host = Optional.of(scmAddresses.iterator().next().getHostName());
}
if (!host.isPresent()) {
throw new IllegalArgumentException(
ScmConfigKeys.OZONE_SCM_SECURITY_SERVICE_ADDRESS_KEY
+ " must be defined. See"
+ " https://wiki.apache.org/hadoop/Ozone#Configuration"
+ " for details on configuring Ozone.");
}
final Optional<Integer> port = getPortNumberFromConfigKeys(conf,
ScmConfigKeys.OZONE_SCM_SECURITY_SERVICE_PORT_KEY);
return NetUtils.createSocketAddr(host.get() + ":" + port
.orElse(ScmConfigKeys.OZONE_SCM_SECURITY_SERVICE_PORT_DEFAULT));
}
/**
* Initialize hadoop metrics system for Ozone servers.
* @param configuration OzoneConfiguration to use.
* @param serverName The logical name of the server components.
* @return
*/
public static MetricsSystem initializeMetrics(
OzoneConfiguration configuration, String serverName) {
MetricsSystem metricsSystem = DefaultMetricsSystem.initialize(serverName);
JvmMetrics.create(serverName,
configuration.get(DFSConfigKeys.DFS_METRICS_SESSION_ID_KEY),
DefaultMetricsSystem.instance());
return metricsSystem;
}
}

107
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/cli/GenericCli.java
View File

@ -1,107 +0,0 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations under
* the License.
*/
package org.apache.hadoop.hdds.cli;
import java.util.HashMap;
import java.util.Map;
import java.util.Map.Entry;
import java.util.concurrent.Callable;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.fs.Path;
import com.google.common.annotations.VisibleForTesting;
import picocli.CommandLine;
import picocli.CommandLine.ExecutionException;
import picocli.CommandLine.Option;
import picocli.CommandLine.RunLast;
/**
* This is a generic parent class for all the ozone related cli tools.
*/
public class GenericCli implements Callable<Void>, GenericParentCommand {
@Option(names = {"--verbose"},
description = "More verbose output. Show the stack trace of the errors.")
private boolean verbose;
@Option(names = {"-D", "--set"})
private Map<String, String> configurationOverrides = new HashMap<>();
@Option(names = {"-conf"})
private String configurationPath;
private final CommandLine cmd;
public GenericCli() {
cmd = new CommandLine(this);
}
public void run(String[] argv) {
try {
execute(argv);
} catch (ExecutionException ex) {
printError(ex.getCause() == null ? ex : ex.getCause());
System.exit(-1);
}
}
@VisibleForTesting
public void execute(String[] argv) {
cmd.parseWithHandler(new RunLast(), argv);
}
protected void printError(Throwable error) {
//message could be null in case of NPE. This is unexpected so we can
//print out the stack trace.
if (verbose || error.getMessage() == null
|| error.getMessage().length() == 0) {
error.printStackTrace(System.err);
} else {
System.err.println(error.getMessage().split("\n")[0]);
}
}
@Override
public Void call() throws Exception {
throw new MissingSubcommandException(cmd);
}
@Override
public OzoneConfiguration createOzoneConfiguration() {
OzoneConfiguration ozoneConf = new OzoneConfiguration();
if (configurationPath != null) {
ozoneConf.addResource(new Path(configurationPath));
}
if (configurationOverrides != null) {
for (Entry<String, String> entry : configurationOverrides.entrySet()) {
ozoneConf.set(entry.getKey(), entry.getValue());
}
}
return ozoneConf;
}
@VisibleForTesting
public picocli.CommandLine getCmd() {
return cmd;
}
@Override
public boolean isVerbose() {
return verbose;
}
}

29
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/cli/GenericParentCommand.java
View File

@ -1,29 +0,0 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations under
* the License.
*/
package org.apache.hadoop.hdds.cli;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
/**
* Interface to access the higher level parameters.
*/
public interface GenericParentCommand {
boolean isVerbose();
OzoneConfiguration createOzoneConfiguration();
}

35
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/cli/HddsVersionProvider.java
View File

@ -1,35 +0,0 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdds.cli;
import org.apache.hadoop.hdds.utils.HddsVersionInfo;
import picocli.CommandLine.IVersionProvider;
/**
* Version provider for the CLI interface.
*/
public class HddsVersionProvider implements IVersionProvider {
@Override
public String[] getVersion() throws Exception {
String[] result = new String[] {
HddsVersionInfo.HDDS_VERSION_INFO.getBuildVersion()
};
return result;
}
}

31
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/cli/MissingSubcommandException.java
View File

@ -1,31 +0,0 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdds.cli;
import picocli.CommandLine;
/**
* Exception to throw if subcommand is not selected but required.
*/
public class MissingSubcommandException extends CommandLine.ParameterException {
public MissingSubcommandException(CommandLine cmd) {
super(cmd, "Incomplete command");
}
}

22
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/cli/package-info.java
View File

@ -1,22 +0,0 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* Generic helper class to make instantiate picocli based cli tools.
*/
package org.apache.hadoop.hdds.cli;

127
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/BlockID.java
View File

@ -1,127 +0,0 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations under
* the License.
*/
package org.apache.hadoop.hdds.client;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import java.util.Objects;
/**
* BlockID of Ozone (containerID + localID + blockCommitSequenceId).
*/
public class BlockID {
private ContainerBlockID containerBlockID;
private long blockCommitSequenceId;
public BlockID(long containerID, long localID) {
this(containerID, localID, 0);
}
private BlockID(long containerID, long localID, long bcsID) {
containerBlockID = new ContainerBlockID(containerID, localID);
blockCommitSequenceId = bcsID;
}
public BlockID(ContainerBlockID containerBlockID) {
this(containerBlockID, 0);
}
private BlockID(ContainerBlockID containerBlockID, long bcsId) {
this.containerBlockID = containerBlockID;
blockCommitSequenceId = bcsId;
}
public long getContainerID() {
return containerBlockID.getContainerID();
}
public long getLocalID() {
return containerBlockID.getLocalID();
}
public long getBlockCommitSequenceId() {
return blockCommitSequenceId;
}
public void setBlockCommitSequenceId(long blockCommitSequenceId) {
this.blockCommitSequenceId = blockCommitSequenceId;
}
public ContainerBlockID getContainerBlockID() {
return containerBlockID;
}
public void setContainerBlockID(ContainerBlockID containerBlockID) {
this.containerBlockID = containerBlockID;
}
@Override
public String toString() {
return new StringBuilder().append(getContainerBlockID().toString())
.append(" bcsId: ")
.append(blockCommitSequenceId)
.toString();
}
public ContainerProtos.DatanodeBlockID getDatanodeBlockIDProtobuf() {
return ContainerProtos.DatanodeBlockID.newBuilder().
setContainerID(containerBlockID.getContainerID())
.setLocalID(containerBlockID.getLocalID())
.setBlockCommitSequenceId(blockCommitSequenceId).build();
}
public static BlockID getFromProtobuf(
ContainerProtos.DatanodeBlockID blockID) {
return new BlockID(blockID.getContainerID(),
blockID.getLocalID(), blockID.getBlockCommitSequenceId());
}
public HddsProtos.BlockID getProtobuf() {
return HddsProtos.BlockID.newBuilder()
.setContainerBlockID(containerBlockID.getProtobuf())
.setBlockCommitSequenceId(blockCommitSequenceId).build();
}
public static BlockID getFromProtobuf(HddsProtos.BlockID blockID) {
return new BlockID(
ContainerBlockID.getFromProtobuf(blockID.getContainerBlockID()),
blockID.getBlockCommitSequenceId());
}
@Override
public boolean equals(Object o) {
if (this == o) {
return true;
}
if (o == null || getClass() != o.getClass()) {
return false;
}
BlockID blockID = (BlockID) o;
return containerBlockID.equals(blockID.getContainerBlockID())
&& blockCommitSequenceId == blockID.getBlockCommitSequenceId();
}
@Override
public int hashCode() {
return Objects
.hash(containerBlockID.getContainerID(), containerBlockID.getLocalID(),
blockCommitSequenceId);
}
}

79
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/ContainerBlockID.java
View File

@ -1,79 +0,0 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations under
* the License.
*/
package org.apache.hadoop.hdds.client;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import java.util.Objects;
/**
* BlockID returned by SCM during allocation of block (containerID + localID).
*/
public class ContainerBlockID {
private long containerID;
private long localID;
public ContainerBlockID(long containerID, long localID) {
this.containerID = containerID;
this.localID = localID;
}
public long getContainerID() {
return containerID;
}
public long getLocalID() {
return localID;
}
@Override
public String toString() {
return new StringBuffer()
.append("conID: ")
.append(containerID)
.append(" locID: ")
.append(localID).toString();
}
public HddsProtos.ContainerBlockID getProtobuf() {
return HddsProtos.ContainerBlockID.newBuilder().
setContainerID(containerID).setLocalID(localID).build();
}
public static ContainerBlockID getFromProtobuf(
HddsProtos.ContainerBlockID containerBlockID) {
return new ContainerBlockID(containerBlockID.getContainerID(),
containerBlockID.getLocalID());
}
@Override
public boolean equals(Object o) {
if (this == o) {
return true;
}
if (o == null || getClass() != o.getClass()) {
return false;
}
ContainerBlockID blockID = (ContainerBlockID) o;
return containerID == blockID.containerID && localID == blockID.localID;
}
@Override
public int hashCode() {
return Objects.hash(containerID, localID);
}
}

203
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/OzoneQuota.java
View File

@ -1,203 +0,0 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdds.client;
import org.apache.hadoop.ozone.OzoneConsts;
/**
* represents an OzoneQuota Object that can be applied to
* a storage volume.
*/
public class OzoneQuota {
public static final String OZONE_QUOTA_BYTES = "BYTES";
public static final String OZONE_QUOTA_MB = "MB";
public static final String OZONE_QUOTA_GB = "GB";
public static final String OZONE_QUOTA_TB = "TB";
private Units unit;
private long size;
/** Quota Units.*/
public enum Units {UNDEFINED, BYTES, KB, MB, GB, TB}
/**
* Returns size.
*
* @return long
*/
public long getSize() {
return size;
}
/**
* Returns Units.
*
* @return Unit in MB, GB or TB
*/
public Units getUnit() {
return unit;
}
/**
* Constructs a default Quota object.
*/
public OzoneQuota() {
this.size = 0;
this.unit = Units.UNDEFINED;
}
/**
* Constructor for Ozone Quota.
*
* @param size Long Size
* @param unit MB, GB or TB
*/
public OzoneQuota(long size, Units unit) {
this.size = size;
this.unit = unit;
}
/**
* Formats a quota as a string.
*
* @param quota the quota to format
* @return string representation of quota
*/
public static String formatQuota(OzoneQuota quota) {
return String.valueOf(quota.size) + quota.unit;
}
/**
* Parses a user provided string and returns the
* Quota Object.
*
* @param quotaString Quota String
*
* @return OzoneQuota object
*
* @throws IllegalArgumentException
*/
public static OzoneQuota parseQuota(String quotaString)
throws IllegalArgumentException {
if ((quotaString == null) || (quotaString.isEmpty())) {
throw new IllegalArgumentException(
"Quota string cannot be null or empty.");
}
String uppercase = quotaString.toUpperCase().replaceAll("\\s+", "");
String size = "";
int nSize;
Units currUnit = Units.MB;
Boolean found = false;
if (uppercase.endsWith(OZONE_QUOTA_MB)) {
size = uppercase
.substring(0, uppercase.length() - OZONE_QUOTA_MB.length());
currUnit = Units.MB;
found = true;
}
if (uppercase.endsWith(OZONE_QUOTA_GB)) {
size = uppercase
.substring(0, uppercase.length() - OZONE_QUOTA_GB.length());
currUnit = Units.GB;
found = true;
}
if (uppercase.endsWith(OZONE_QUOTA_TB)) {
size = uppercase
.substring(0, uppercase.length() - OZONE_QUOTA_TB.length());
currUnit = Units.TB;
found = true;
}
if (uppercase.endsWith(OZONE_QUOTA_BYTES)) {
size = uppercase
.substring(0, uppercase.length() - OZONE_QUOTA_BYTES.length());
currUnit = Units.BYTES;
found = true;
}
if (!found) {
throw new IllegalArgumentException(
"Quota unit not recognized. Supported values are BYTES, MB, GB and " +
"TB.");
}
nSize = Integer.parseInt(size);
if (nSize < 0) {
throw new IllegalArgumentException("Quota cannot be negative.");
}
return new OzoneQuota(nSize, currUnit);
}
/**
* Returns size in Bytes or -1 if there is no Quota.
*/
public long sizeInBytes() {
switch (this.unit) {
case BYTES:
return this.getSize();
case MB:
return this.getSize() * OzoneConsts.MB;
case GB:
return this.getSize() * OzoneConsts.GB;
case TB:
return this.getSize() * OzoneConsts.TB;
case UNDEFINED:
default:
return -1;
}
}
/**
* Returns OzoneQuota corresponding to size in bytes.
*
* @param sizeInBytes size in bytes to be converted
*
* @return OzoneQuota object
*/
public static OzoneQuota getOzoneQuota(long sizeInBytes) {
long size;
Units unit;
if (sizeInBytes % OzoneConsts.TB == 0) {
size = sizeInBytes / OzoneConsts.TB;
unit = Units.TB;
} else if (sizeInBytes % OzoneConsts.GB == 0) {
size = sizeInBytes / OzoneConsts.GB;
unit = Units.GB;
} else if (sizeInBytes % OzoneConsts.MB == 0) {
size = sizeInBytes / OzoneConsts.MB;
unit = Units.MB;
} else {
size = sizeInBytes;
unit = Units.BYTES;
}
return new OzoneQuota((int)size, unit);
}
@Override
public String toString() {
return size + " " + unit;
}
}

81
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/ReplicationFactor.java
View File

@ -1,81 +0,0 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdds.client;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
/**
* The replication factor to be used while writing key into ozone.
*/
public enum ReplicationFactor {
ONE(1),
THREE(3);
/**
* Integer representation of replication.
*/
private int value;
/**
* Initializes ReplicationFactor with value.
* @param value replication value
*/
ReplicationFactor(int value) {
this.value = value;
}
/**
* Returns enum value corresponding to the int value.
* @param value replication value
* @return ReplicationFactor
*/
public static ReplicationFactor valueOf(int value) {
if(value == 1) {
return ONE;
}
if (value == 3) {
return THREE;
}
throw new IllegalArgumentException("Unsupported value: " + value);
}
public static ReplicationFactor fromProto(
HddsProtos.ReplicationFactor replicationFactor) {
if (replicationFactor == null) {
return null;
}
switch (replicationFactor) {
case ONE:
return ReplicationFactor.ONE;
case THREE:
return ReplicationFactor.THREE;
default:
throw new IllegalArgumentException(
"Unsupported ProtoBuf replication factor: " + replicationFactor);
}
}
/**
* Returns integer representation of ReplicationFactor.
* @return replication value
*/
public int getValue() {
return value;
}
}

48
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/ReplicationType.java
View File

@ -1,48 +0,0 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdds.client;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
/**
* The replication type to be used while writing key into ozone.
*/
public enum ReplicationType {
RATIS,
STAND_ALONE,
CHAINED;
public static ReplicationType fromProto(
HddsProtos.ReplicationType replicationType) {
if (replicationType == null) {
return null;
}
switch (replicationType) {
case RATIS:
return ReplicationType.RATIS;
case STAND_ALONE:
return ReplicationType.STAND_ALONE;
case CHAINED:
return ReplicationType.CHAINED;
default:
throw new IllegalArgumentException(
"Unsupported ProtoBuf replication type: " + replicationType);
}
}
}

23
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/package-info.java
View File

@ -1,23 +0,0 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdds.client;
/**
* Base property types for HDDS containers and replications.
*/

190
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/HddsConfServlet.java
View File

@ -1,190 +0,0 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdds.conf;
import com.google.gson.Gson;
import java.io.IOException;
import java.io.Writer;
import java.util.HashMap;
import java.util.Map;
import java.util.Properties;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import javax.ws.rs.core.HttpHeaders;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.http.HttpServer2;
import com.google.common.annotations.VisibleForTesting;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_TAGS_SYSTEM_KEY;
/**
* A servlet to print out the running configuration data.
*/
@InterfaceAudience.LimitedPrivate({"HDFS", "MapReduce"})
@InterfaceStability.Unstable
public class HddsConfServlet extends HttpServlet {
private static final long serialVersionUID = 1L;
protected static final String FORMAT_JSON = "json";
protected static final String FORMAT_XML = "xml";
private static final String COMMAND = "cmd";
private static final OzoneConfiguration OZONE_CONFIG =
new OzoneConfiguration();
private static final transient Logger LOG =
LoggerFactory.getLogger(HddsConfServlet.class);
/**
* Return the Configuration of the daemon hosting this servlet.
* This is populated when the HttpServer starts.
*/
private Configuration getConfFromContext() {
Configuration conf = (Configuration) getServletContext().getAttribute(
HttpServer2.CONF_CONTEXT_ATTRIBUTE);
assert conf != null;
return conf;
}
@Override
public void doGet(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException {
if (!HttpServer2.isInstrumentationAccessAllowed(getServletContext(),
request, response)) {
return;
}
String format = parseAcceptHeader(request);
if (FORMAT_XML.equals(format)) {
response.setContentType("text/xml; charset=utf-8");
} else if (FORMAT_JSON.equals(format)) {
response.setContentType("application/json; charset=utf-8");
}
String name = request.getParameter("name");
Writer out = response.getWriter();
String cmd = request.getParameter(COMMAND);
processCommand(cmd, format, request, response, out, name);
out.close();
}
private void processCommand(String cmd, String format,
HttpServletRequest request, HttpServletResponse response, Writer out,
String name)
throws IOException {
try {
if (cmd == null) {
if (FORMAT_XML.equals(format)) {
response.setContentType("text/xml; charset=utf-8");
} else if (FORMAT_JSON.equals(format)) {
response.setContentType("application/json; charset=utf-8");
}
writeResponse(getConfFromContext(), out, format, name);
} else {
processConfigTagRequest(request, out);
}
} catch (BadFormatException bfe) {
response.sendError(HttpServletResponse.SC_BAD_REQUEST, bfe.getMessage());
} catch (IllegalArgumentException iae) {
response.sendError(HttpServletResponse.SC_NOT_FOUND, iae.getMessage());
}
}
@VisibleForTesting
static String parseAcceptHeader(HttpServletRequest request) {
String format = request.getHeader(HttpHeaders.ACCEPT);
return format != null && format.contains(FORMAT_JSON) ?
FORMAT_JSON : FORMAT_XML;
}
/**
* Guts of the servlet - extracted for easy testing.
*/
static void writeResponse(Configuration conf,
Writer out, String format, String propertyName)
throws IOException, IllegalArgumentException, BadFormatException {
if (FORMAT_JSON.equals(format)) {
Configuration.dumpConfiguration(conf, propertyName, out);
} else if (FORMAT_XML.equals(format)) {
conf.writeXml(propertyName, out);
} else {
throw new BadFormatException("Bad format: " + format);
}
}
/**
* Exception for signal bad content type.
*/
public static class BadFormatException extends Exception {
private static final long serialVersionUID = 1L;
public BadFormatException(String msg) {
super(msg);
}
}
private void processConfigTagRequest(HttpServletRequest request,
Writer out) throws IOException {
String cmd = request.getParameter(COMMAND);
Gson gson = new Gson();
Configuration config = getOzoneConfig();
switch (cmd) {
case "getOzoneTags":
out.write(gson.toJson(config.get(OZONE_TAGS_SYSTEM_KEY)
.split(",")));
break;
case "getPropertyByTag":
String tags = request.getParameter("tags");
Map<String, Properties> propMap = new HashMap<>();
for (String tag : tags.split(",")) {
if (config.isPropertyTag(tag)) {
Properties properties = config.getAllPropertiesByTag(tag);
propMap.put(tag, properties);
} else {
if (LOG.isDebugEnabled()) {
LOG.debug("Not a valid tag" + tag);
}
}
}
out.write(gson.toJsonTree(propMap).toString());
break;
default:
throw new IllegalArgumentException(cmd + " is not a valid command.");
}
}
private static Configuration getOzoneConfig() {
return OZONE_CONFIG;
}
}

328
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/OzoneConfiguration.java
View File

@ -1,328 +0,0 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdds.conf;
import javax.xml.bind.JAXBContext;
import javax.xml.bind.JAXBException;
import javax.xml.bind.Unmarshaller;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlRootElement;
import java.io.IOException;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.net.URL;
import java.util.ArrayList;
import java.util.Enumeration;
import java.util.List;
import java.util.Properties;
import com.google.common.base.Preconditions;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
/**
* Configuration for ozone.
*/
@InterfaceAudience.Private
public class OzoneConfiguration extends Configuration {
static {
activate();
}
public static OzoneConfiguration of(Configuration conf) {
Preconditions.checkNotNull(conf);
return conf instanceof OzoneConfiguration
? (OzoneConfiguration) conf
: new OzoneConfiguration(conf);
}
public OzoneConfiguration() {
OzoneConfiguration.activate();
loadDefaults();
}
public OzoneConfiguration(Configuration conf) {
super(conf);
//load the configuration from the classloader of the original conf.
setClassLoader(conf.getClassLoader());
if (!(conf instanceof OzoneConfiguration)) {
loadDefaults();
}
}
private void loadDefaults() {
try {
//there could be multiple ozone-default-generated.xml files on the
// classpath, which are generated by the annotation processor.
// Here we add all of them to the list of the available configuration.
Enumeration<URL> generatedDefaults =
OzoneConfiguration.class.getClassLoader().getResources(
"ozone-default-generated.xml");
while (generatedDefaults.hasMoreElements()) {
addResource(generatedDefaults.nextElement());
}
} catch (IOException e) {
e.printStackTrace();
}
addResource("ozone-site.xml");
}
public List<Property> readPropertyFromXml(URL url) throws JAXBException {
JAXBContext context = JAXBContext.newInstance(XMLConfiguration.class);
Unmarshaller um = context.createUnmarshaller();
XMLConfiguration config = (XMLConfiguration) um.unmarshal(url);
return config.getProperties();
}
/**
* Create a Configuration object and inject the required configuration values.
*
* @param configurationClass The class where the fields are annotated with
* the configuration.
* @return Initiated java object where the config fields are injected.
*/
public <T> T getObject(Class<T> configurationClass) {
T configuration;
try {
configuration = configurationClass.newInstance();
} catch (InstantiationException | IllegalAccessException e) {
throw new ConfigurationException(
"Configuration class can't be created: " + configurationClass, e);
}
ConfigGroup configGroup =
configurationClass.getAnnotation(ConfigGroup.class);
String prefix = configGroup.prefix();
for (Method setterMethod : configurationClass.getMethods()) {
if (setterMethod.isAnnotationPresent(Config.class)) {
String methodLocation =
configurationClass + "." + setterMethod.getName();
Config configAnnotation = setterMethod.getAnnotation(Config.class);
String key = prefix + "." + configAnnotation.key();
Class<?>[] parameterTypes = setterMethod.getParameterTypes();
if (parameterTypes.length != 1) {
throw new ConfigurationException(
"@Config annotation should be used on simple setter: "
+ methodLocation);
}
ConfigType type = configAnnotation.type();
if (type == ConfigType.AUTO) {
type = detectConfigType(parameterTypes[0], methodLocation);
}
//Note: default value is handled by ozone-default.xml. Here we can
//use any default.
try {
switch (type) {
case STRING:
setterMethod.invoke(configuration, get(key));
break;
case INT:
setterMethod.invoke(configuration,
getInt(key, 0));
break;
case BOOLEAN:
setterMethod.invoke(configuration,
getBoolean(key, false));
break;
case LONG:
setterMethod.invoke(configuration,
getLong(key, 0));
break;
case TIME:
setterMethod.invoke(configuration,
getTimeDuration(key, 0, configAnnotation.timeUnit()));
break;
default:
throw new ConfigurationException(
"Unsupported ConfigType " + type + " on " + methodLocation);
}
} catch (InvocationTargetException | IllegalAccessException e) {
throw new ConfigurationException(
"Can't inject configuration to " + methodLocation, e);
}
}
}
return configuration;
}
private ConfigType detectConfigType(Class<?> parameterType,
String methodLocation) {
ConfigType type;
if (parameterType == String.class) {
type = ConfigType.STRING;
} else if (parameterType == Integer.class || parameterType == int.class) {
type = ConfigType.INT;
} else if (parameterType == Long.class || parameterType == long.class) {
type = ConfigType.LONG;
} else if (parameterType == Boolean.class
|| parameterType == boolean.class) {
type = ConfigType.BOOLEAN;
} else {
throw new ConfigurationException(
"Unsupported configuration type " + parameterType + " in "
+ methodLocation);
}
return type;
}
/**
* Class to marshall/un-marshall configuration from xml files.
*/
@XmlAccessorType(XmlAccessType.FIELD)
@XmlRootElement(name = "configuration")
public static class XMLConfiguration {
@XmlElement(name = "property", type = Property.class)
private List<Property> properties = new ArrayList<>();
public XMLConfiguration() {
}
public XMLConfiguration(List<Property> properties) {
this.properties = properties;
}
public List<Property> getProperties() {
return properties;
}
public void setProperties(List<Property> properties) {
this.properties = properties;
}
}
/**
* Class to marshall/un-marshall configuration properties from xml files.
*/
@XmlAccessorType(XmlAccessType.FIELD)
@XmlRootElement(name = "property")
public static class Property implements Comparable<Property> {
private String name;
private String value;
private String tag;
private String description;
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public String getValue() {
return value;
}
public void setValue(String value) {
this.value = value;
}
public String getTag() {
return tag;
}
public void setTag(String tag) {
this.tag = tag;
}
public String getDescription() {
return description;
}
public void setDescription(String description) {
this.description = description;
}
@Override
public int compareTo(Property o) {
if (this == o) {
return 0;
}
return this.getName().compareTo(o.getName());
}
@Override
public String toString() {
return this.getName() + " " + this.getValue() + " " + this.getTag();
}
@Override
public int hashCode() {
return this.getName().hashCode();
}
@Override
public boolean equals(Object obj) {
return (obj instanceof Property) && (((Property) obj).getName())
.equals(this.getName());
}
}
public static void activate() {
// adds the default resources
Configuration.addDefaultResource("hdfs-default.xml");
Configuration.addDefaultResource("hdfs-site.xml");
Configuration.addDefaultResource("ozone-default.xml");
}
/**
* The super class method getAllPropertiesByTag
* does not override values of properties
* if there is no tag present in the configs of
* newly added resources.
*
* @param tag
* @return Properties that belong to the tag
*/
@Override
public Properties getAllPropertiesByTag(String tag) {
// Call getProps first to load the newly added resources
// before calling super.getAllPropertiesByTag
Properties updatedProps = getProps();
Properties propertiesByTag = super.getAllPropertiesByTag(tag);
Properties props = new Properties();
Enumeration properties = propertiesByTag.propertyNames();
while (properties.hasMoreElements()) {
Object propertyName = properties.nextElement();
// get the current value of the property
Object value = updatedProps.getProperty(propertyName.toString());
if (value != null) {
props.put(propertyName, value);
}
}
return props;
}
}

18
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/package-info.java
View File

@ -1,18 +0,0 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdds.conf;

36
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/function/FunctionWithServiceException.java
View File

@ -1,36 +0,0 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdds.function;
import com.google.protobuf.ServiceException;
/**
* Functional interface like java.util.function.Function but with
* checked exception.
*/
@FunctionalInterface
public interface FunctionWithServiceException<T, R> {
/**
* Applies this function to the given argument.
*
* @param t the function argument
* @return the function result
*/
R apply(T t) throws ServiceException;
}

22
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/function/package-info.java
View File

@ -1,22 +0,0 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* Functional interfaces for ozone, similar to java.util.function.
*/
package org.apache.hadoop.hdds.function;

23
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/package-info.java
View File

@ -1,23 +0,0 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdds;
/**
* Generic HDDS specific configurator and helper classes.
*/

493
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocol/DatanodeDetails.java
View File

@ -1,493 +0,0 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdds.protocol;
import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.scm.net.NetConstants;
import org.apache.hadoop.hdds.scm.net.NodeImpl;
import java.util.ArrayList;
import java.util.List;
import java.util.UUID;
/**
* DatanodeDetails class contains details about DataNode like:
* - UUID of the DataNode.
* - IP and Hostname details.
* - Port details to which the DataNode will be listening.
*/
@InterfaceAudience.Private
@InterfaceStability.Evolving
public class DatanodeDetails extends NodeImpl implements
Comparable<DatanodeDetails> {
/**
* DataNode's unique identifier in the cluster.
*/
private final UUID uuid;
private String ipAddress;
private String hostName;
private List<Port> ports;
private String certSerialId;
/**
* Constructs DatanodeDetails instance. DatanodeDetails.Builder is used
* for instantiating DatanodeDetails.
* @param uuid DataNode's UUID
* @param ipAddress IP Address of this DataNode
* @param hostName DataNode's hostname
* @param networkLocation DataNode's network location path
* @param ports Ports used by the DataNode
* @param certSerialId serial id from SCM issued certificate.
*/
private DatanodeDetails(String uuid, String ipAddress, String hostName,
String networkLocation, List<Port> ports, String certSerialId) {
super(hostName, networkLocation, NetConstants.NODE_COST_DEFAULT);
this.uuid = UUID.fromString(uuid);
this.ipAddress = ipAddress;
this.hostName = hostName;
this.ports = ports;
this.certSerialId = certSerialId;
}
protected DatanodeDetails(DatanodeDetails datanodeDetails) {
super(datanodeDetails.getHostName(), datanodeDetails.getNetworkLocation(),
datanodeDetails.getCost());
this.uuid = datanodeDetails.uuid;
this.ipAddress = datanodeDetails.ipAddress;
this.hostName = datanodeDetails.hostName;
this.ports = datanodeDetails.ports;
this.setNetworkName(datanodeDetails.getNetworkName());
}
/**
* Returns the DataNode UUID.
*
* @return UUID of DataNode
*/
public UUID getUuid() {
return uuid;
}
/**
* Returns the string representation of DataNode UUID.
*
* @return UUID of DataNode
*/
public String getUuidString() {
return uuid.toString();
}
/**
* Sets the IP address of Datanode.
*
* @param ip IP Address
*/
public void setIpAddress(String ip) {
this.ipAddress = ip;
}
/**
* Returns IP address of DataNode.
*
* @return IP address
*/
public String getIpAddress() {
return ipAddress;
}
/**
* Sets the Datanode hostname.
*
* @param host hostname
*/
public void setHostName(String host) {
this.hostName = host;
}
/**
* Returns Hostname of DataNode.
*
* @return Hostname
*/
public String getHostName() {
return hostName;
}
/**
* Sets a DataNode Port.
*
* @param port DataNode port
*/
public void setPort(Port port) {
// If the port is already in the list remove it first and add the
// new/updated port value.
ports.remove(port);
ports.add(port);
}
/**
* Returns all the Ports used by DataNode.
*
* @return DataNode Ports
*/
public List<Port> getPorts() {
return ports;
}
/**
* Given the name returns port number, null if the asked port is not found.
*
* @param name Name of the port
*
* @return Port
*/
public Port getPort(Port.Name name) {
for (Port port : ports) {
if (port.getName().equals(name)) {
return port;
}
}
return null;
}
/**
* Returns a DatanodeDetails from the protocol buffers.
*
* @param datanodeDetailsProto - protoBuf Message
* @return DatanodeDetails
*/
public static DatanodeDetails getFromProtoBuf(
HddsProtos.DatanodeDetailsProto datanodeDetailsProto) {
DatanodeDetails.Builder builder = newBuilder();
builder.setUuid(datanodeDetailsProto.getUuid());
if (datanodeDetailsProto.hasIpAddress()) {
builder.setIpAddress(datanodeDetailsProto.getIpAddress());
}
if (datanodeDetailsProto.hasHostName()) {
builder.setHostName(datanodeDetailsProto.getHostName());
}
if (datanodeDetailsProto.hasCertSerialId()) {
builder.setCertSerialId(datanodeDetailsProto.getCertSerialId());
}
for (HddsProtos.Port port : datanodeDetailsProto.getPortsList()) {
builder.addPort(newPort(
Port.Name.valueOf(port.getName().toUpperCase()), port.getValue()));
}
if (datanodeDetailsProto.hasNetworkName()) {
builder.setNetworkName(datanodeDetailsProto.getNetworkName());
}
if (datanodeDetailsProto.hasNetworkLocation()) {
builder.setNetworkLocation(datanodeDetailsProto.getNetworkLocation());
}
return builder.build();
}
/**
* Returns a DatanodeDetails protobuf message from a datanode ID.
* @return HddsProtos.DatanodeDetailsProto
*/
public HddsProtos.DatanodeDetailsProto getProtoBufMessage() {
HddsProtos.DatanodeDetailsProto.Builder builder =
HddsProtos.DatanodeDetailsProto.newBuilder()
.setUuid(getUuidString());
if (ipAddress != null) {
builder.setIpAddress(ipAddress);
}
if (hostName != null) {
builder.setHostName(hostName);
}
if (certSerialId != null) {
builder.setCertSerialId(certSerialId);
}
if (!Strings.isNullOrEmpty(getNetworkName())) {
builder.setNetworkName(getNetworkName());
}
if (!Strings.isNullOrEmpty(getNetworkLocation())) {
builder.setNetworkLocation(getNetworkLocation());
}
for (Port port : ports) {
builder.addPorts(HddsProtos.Port.newBuilder()
.setName(port.getName().toString())
.setValue(port.getValue())
.build());
}
return builder.build();
}
@Override
public String toString() {
return uuid.toString() + "{" +
"ip: " +
ipAddress +
", host: " +
hostName +
", networkLocation: " +
getNetworkLocation() +
", certSerialId: " + certSerialId +
"}";
}
@Override
public int compareTo(DatanodeDetails that) {
return this.getUuid().compareTo(that.getUuid());
}
@Override
public boolean equals(Object obj) {
return obj instanceof DatanodeDetails &&
uuid.equals(((DatanodeDetails) obj).uuid);
}
@Override
public int hashCode() {
return uuid.hashCode();
}
/**
* Returns DatanodeDetails.Builder instance.
*
* @return DatanodeDetails.Builder
*/
public static Builder newBuilder() {
return new Builder();
}
/**
* Builder class for building DatanodeDetails.
*/
public static final class Builder {
private String id;
private String ipAddress;
private String hostName;
private String networkName;
private String networkLocation;
private List<Port> ports;
private String certSerialId;
/**
* Default private constructor. To create Builder instance use
* DatanodeDetails#newBuilder.
*/
private Builder() {
ports = new ArrayList<>();
}
/**
* Sets the DatanodeUuid.
*
* @param uuid DatanodeUuid
* @return DatanodeDetails.Builder
*/
public Builder setUuid(String uuid) {
this.id = uuid;
return this;
}
/**
* Sets the IP address of DataNode.
*
* @param ip address
* @return DatanodeDetails.Builder
*/
public Builder setIpAddress(String ip) {
this.ipAddress = ip;
return this;
}
/**
* Sets the hostname of DataNode.
*
* @param host hostname
* @return DatanodeDetails.Builder
*/
public Builder setHostName(String host) {
this.hostName = host;
return this;
}
/**
* Sets the network name of DataNode.
*
* @param name network name
* @return DatanodeDetails.Builder
*/
public Builder setNetworkName(String name) {
this.networkName = name;
return this;
}
/**
* Sets the network location of DataNode.
*
* @param loc location
* @return DatanodeDetails.Builder
*/
public Builder setNetworkLocation(String loc) {
this.networkLocation = loc;
return this;
}
/**
* Adds a DataNode Port.
*
* @param port DataNode port
*
* @return DatanodeDetails.Builder
*/
public Builder addPort(Port port) {
this.ports.add(port);
return this;
}
/**
* Adds certificate serial id.
*
* @param certId Serial id of SCM issued certificate.
*
* @return DatanodeDetails.Builder
*/
public Builder setCertSerialId(String certId) {
this.certSerialId = certId;
return this;
}
/**
* Builds and returns DatanodeDetails instance.
*
* @return DatanodeDetails
*/
public DatanodeDetails build() {
Preconditions.checkNotNull(id);
if (networkLocation == null) {
networkLocation = NetConstants.DEFAULT_RACK;
}
DatanodeDetails dn = new DatanodeDetails(id, ipAddress, hostName,
networkLocation, ports, certSerialId);
if (networkName != null) {
dn.setNetworkName(networkName);
}
return dn;
}
}
/**
* Constructs a new Port with name and value.
*
* @param name Name of the port
* @param value Port number
*
* @return {@code Port} instance
*/
public static Port newPort(Port.Name name, Integer value) {
return new Port(name, value);
}
/**
* Container to hold DataNode Port details.
*/
public static final class Port {
/**
* Ports that are supported in DataNode.
*/
public enum Name {
STANDALONE, RATIS, REST
}
private Name name;
private Integer value;
/**
* Private constructor for constructing Port object. Use
* DatanodeDetails#newPort to create a new Port object.
*
* @param name
* @param value
*/
private Port(Name name, Integer value) {
this.name = name;
this.value = value;
}
/**
* Returns the name of the port.
*
* @return Port name
*/
public Name getName() {
return name;
}
/**
* Returns the port number.
*
* @return Port number
*/
public Integer getValue() {
return value;
}
@Override
public int hashCode() {
return name.hashCode();
}
/**
* Ports are considered equal if they have the same name.
*
* @param anObject
* The object to compare this {@code Port} against
* @return {@code true} if the given object represents a {@code Port}
and has the same name, {@code false} otherwise
*/
@Override
public boolean equals(Object anObject) {
if (this == anObject) {
return true;
}
if (anObject instanceof Port) {
return name.equals(((Port) anObject).name);
}
return false;
}
}
/**
* Returns serial id of SCM issued certificate.
*
* @return certificate serial id
*/
public String getCertSerialId() {
return certSerialId;
}
/**
* Set certificate serial id of SCM issued certificate.
*
*/
public void setCertSerialId(String certSerialId) {
this.certSerialId = certSerialId;
}
}

80
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocol/SCMSecurityProtocol.java
View File

@ -1,80 +0,0 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations under
* the License.
*/
package org.apache.hadoop.hdds.protocol;
import java.io.IOException;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.DatanodeDetailsProto;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.OzoneManagerDetailsProto;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.security.KerberosInfo;
/**
* The protocol used to perform security related operations with SCM.
*/
@KerberosInfo(
serverPrincipal = ScmConfigKeys.HDDS_SCM_KERBEROS_PRINCIPAL_KEY)
@InterfaceAudience.Private
public interface SCMSecurityProtocol {
@SuppressWarnings("checkstyle:ConstantName")
/**
* Version 1: Initial version.
*/
long versionID = 1L;
/**
* Get SCM signed certificate for DataNode.
*
* @param dataNodeDetails - DataNode Details.
* @param certSignReq - Certificate signing request.
* @return byte[] - SCM signed certificate.
*/
String getDataNodeCertificate(
DatanodeDetailsProto dataNodeDetails,
String certSignReq) throws IOException;
/**
* Get SCM signed certificate for OM.
*
* @param omDetails - DataNode Details.
* @param certSignReq - Certificate signing request.
* @return String - pem encoded SCM signed
* certificate.
*/
String getOMCertificate(OzoneManagerDetailsProto omDetails,
String certSignReq) throws IOException;
/**
* Get SCM signed certificate for given certificate serial id if it exists.
* Throws exception if it's not found.
*
* @param certSerialId - Certificate serial id.
* @return String - pem encoded SCM signed
* certificate with given cert id if it
* exists.
*/
String getCertificate(String certSerialId) throws IOException;
/**
* Get CA certificate.
*
* @return String - pem encoded CA certificate.
*/
String getCACertificate() throws IOException;
}

22
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocol/package-info.java
View File

@ -1,22 +0,0 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* This package contains HDDS protocol related classes.
*/
package org.apache.hadoop.hdds.protocol;

213
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocolPB/SCMSecurityProtocolClientSideTranslatorPB.java
View File

@ -1,213 +0,0 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations under
* the License.
*/
package org.apache.hadoop.hdds.protocolPB;
import java.io.Closeable;
import java.io.IOException;
import java.util.function.Consumer;
import org.apache.hadoop.hdds.protocol.SCMSecurityProtocol;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.DatanodeDetailsProto;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.OzoneManagerDetailsProto;
import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos;
import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMGetCACertificateRequestProto;
import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMGetCertResponseProto;
import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMGetCertificateRequestProto;
import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMGetDataNodeCertRequestProto;
import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMSecurityRequest;
import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMSecurityRequest.Builder;
import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMSecurityResponse;
import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.Type;
import org.apache.hadoop.hdds.tracing.TracingUtil;
import org.apache.hadoop.ipc.ProtobufHelper;
import org.apache.hadoop.ipc.ProtocolTranslator;
import org.apache.hadoop.ipc.RPC;
import com.google.protobuf.RpcController;
import com.google.protobuf.ServiceException;
import static org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMGetOMCertRequestProto;
/**
* This class is the client-side translator that forwards requests for
* {@link SCMSecurityProtocol} to the {@link SCMSecurityProtocolPB} proxy.
*/
public class SCMSecurityProtocolClientSideTranslatorPB implements
SCMSecurityProtocol, ProtocolTranslator, Closeable {
/**
* RpcController is not used and hence is set to null.
*/
private static final RpcController NULL_RPC_CONTROLLER = null;
private final SCMSecurityProtocolPB rpcProxy;
public SCMSecurityProtocolClientSideTranslatorPB(
SCMSecurityProtocolPB rpcProxy) {
this.rpcProxy = rpcProxy;
}
/**
* Helper method to wrap the request and send the message.
*/
private SCMSecurityResponse submitRequest(
SCMSecurityProtocolProtos.Type type,
Consumer<Builder> builderConsumer) throws IOException {
final SCMSecurityResponse response;
try {
Builder builder = SCMSecurityRequest.newBuilder()
.setCmdType(type)
.setTraceID(TracingUtil.exportCurrentSpan());
builderConsumer.accept(builder);
SCMSecurityRequest wrapper = builder.build();
response = rpcProxy.submitRequest(NULL_RPC_CONTROLLER, wrapper);
} catch (ServiceException ex) {
throw ProtobufHelper.getRemoteException(ex);
}
return response;
}
/**
* Closes this stream and releases any system resources associated
* with it. If the stream is already closed then invoking this
* method has no effect.
*
* <p> As noted in {@link AutoCloseable#close()}, cases where the
* close may fail require careful attention. It is strongly advised
* to relinquish the underlying resources and to internally
* <em>mark</em> the {@code Closeable} as closed, prior to throwing
* the {@code IOException}.
*
* @throws IOException if an I/O error occurs
*/
@Override
public void close() throws IOException {
RPC.stopProxy(rpcProxy);
}
/**
* Get SCM signed certificate for DataNode.
*
* @param dataNodeDetails - DataNode Details.
* @param certSignReq - Certificate signing request.
* @return byte[] - SCM signed certificate.
*/
@Override
public String getDataNodeCertificate(DatanodeDetailsProto dataNodeDetails,
String certSignReq) throws IOException {
return getDataNodeCertificateChain(dataNodeDetails, certSignReq)
.getX509Certificate();
}
/**
* Get SCM signed certificate for OM.
*
* @param omDetails - OzoneManager Details.
* @param certSignReq - Certificate signing request.
* @return byte[] - SCM signed certificate.
*/
@Override
public String getOMCertificate(OzoneManagerDetailsProto omDetails,
String certSignReq) throws IOException {
return getOMCertChain(omDetails, certSignReq).getX509Certificate();
}
/**
* Get SCM signed certificate for OM.
*
* @param omDetails - OzoneManager Details.
* @param certSignReq - Certificate signing request.
* @return byte[] - SCM signed certificate.
*/
public SCMGetCertResponseProto getOMCertChain(
OzoneManagerDetailsProto omDetails, String certSignReq)
throws IOException {
SCMGetOMCertRequestProto request = SCMGetOMCertRequestProto
.newBuilder()
.setCSR(certSignReq)
.setOmDetails(omDetails)
.build();
return submitRequest(Type.GetOMCertificate,
builder -> builder.setGetOMCertRequest(request))
.getGetCertResponseProto();
}
/**
* Get SCM signed certificate with given serial id. Throws exception if
* certificate is not found.
*
* @param certSerialId - Certificate serial id.
* @return string - pem encoded certificate.
*/
@Override
public String getCertificate(String certSerialId) throws IOException {
SCMGetCertificateRequestProto request = SCMGetCertificateRequestProto
.newBuilder()
.setCertSerialId(certSerialId)
.build();
return submitRequest(Type.GetCertificate,
builder -> builder.setGetCertificateRequest(request))
.getGetCertResponseProto()
.getX509Certificate();
}
/**
* Get SCM signed certificate for Datanode.
*
* @param dnDetails - Datanode Details.
* @param certSignReq - Certificate signing request.
* @return byte[] - SCM signed certificate.
*/
public SCMGetCertResponseProto getDataNodeCertificateChain(
DatanodeDetailsProto dnDetails, String certSignReq)
throws IOException {
SCMGetDataNodeCertRequestProto request =
SCMGetDataNodeCertRequestProto.newBuilder()
.setCSR(certSignReq)
.setDatanodeDetails(dnDetails)
.build();
return submitRequest(Type.GetDataNodeCertificate,
builder -> builder.setGetDataNodeCertRequest(request))
.getGetCertResponseProto();
}
/**
* Get CA certificate.
*
* @return serial - Root certificate.
*/
@Override
public String getCACertificate() throws IOException {
SCMGetCACertificateRequestProto protoIns = SCMGetCACertificateRequestProto
.getDefaultInstance();
return submitRequest(Type.GetCACertificate,
builder -> builder.setGetCACertificateRequest(protoIns))
.getGetCertResponseProto().getX509Certificate();
}
/**
* Return the proxy object underlying this protocol translator.
*
* @return the proxy object underlying this protocol translator.
*/
@Override
public Object getUnderlyingProxyObject() {
return rpcProxy;
}
}

35
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocolPB/SCMSecurityProtocolPB.java
View File

@ -1,35 +0,0 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* <p/>
* http://www.apache.org/licenses/LICENSE-2.0
* <p/>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations under
* the License.
*/
package org.apache.hadoop.hdds.protocolPB;
import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMSecurityProtocolService;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.ipc.ProtocolInfo;
import org.apache.hadoop.security.KerberosInfo;
/**
* Protocol for security related operations on SCM.
*/
@ProtocolInfo(protocolName =
"org.apache.hadoop.hdds.protocol.SCMSecurityProtocol",
protocolVersion = 1)
@KerberosInfo(serverPrincipal = ScmConfigKeys.HDDS_SCM_KERBEROS_PRINCIPAL_KEY)
public interface SCMSecurityProtocolPB extends
SCMSecurityProtocolService.BlockingInterface {
}

22
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocolPB/package-info.java
View File

@ -1,22 +0,0 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdds.protocolPB;
/**
* This package contains classes for wiring HDDS protobuf calls to rpc.
*/

107
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ratis/ContainerCommandRequestMessage.java
View File

@ -1,107 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdds.ratis;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandRequestProto;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.PutSmallFileRequestProto;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Type;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.WriteChunkRequestProto;
import org.apache.ratis.protocol.Message;
import org.apache.ratis.protocol.RaftGroupId;
import org.apache.ratis.thirdparty.com.google.protobuf.ByteString;
import org.apache.ratis.thirdparty.com.google.protobuf.InvalidProtocolBufferException;
import org.apache.ratis.util.JavaUtils;
import java.util.Objects;
import java.util.function.Supplier;
/**
* Implementing the {@link Message} interface
* for {@link ContainerCommandRequestProto}.
*/
public final class ContainerCommandRequestMessage implements Message {
public static ContainerCommandRequestMessage toMessage(
ContainerCommandRequestProto request, String traceId) {
final ContainerCommandRequestProto.Builder b
= ContainerCommandRequestProto.newBuilder(request);
if (traceId != null) {
b.setTraceID(traceId);
}
ByteString data = ByteString.EMPTY;
if (request.getCmdType() == Type.WriteChunk) {
final WriteChunkRequestProto w = request.getWriteChunk();
data = w.getData();
b.setWriteChunk(w.toBuilder().clearData());
} else if (request.getCmdType() == Type.PutSmallFile) {
final PutSmallFileRequestProto p = request.getPutSmallFile();
data = p.getData();
b.setPutSmallFile(p.toBuilder().setData(ByteString.EMPTY));
}
return new ContainerCommandRequestMessage(b.build(), data);
}
public static ContainerCommandRequestProto toProto(
ByteString bytes, RaftGroupId groupId)
throws InvalidProtocolBufferException {
final int i = 4 + bytes.asReadOnlyByteBuffer().getInt();
final ContainerCommandRequestProto header
= ContainerCommandRequestProto.parseFrom(bytes.substring(4, i));
// TODO: setting pipeline id can be avoided if the client is sending it.
// In such case, just have to validate the pipeline id.
final ContainerCommandRequestProto.Builder b = header.toBuilder();
if (groupId != null) {
b.setPipelineID(groupId.getUuid().toString());
}
final ByteString data = bytes.substring(i);
if (header.getCmdType() == Type.WriteChunk) {
b.setWriteChunk(b.getWriteChunkBuilder().setData(data));
} else if (header.getCmdType() == Type.PutSmallFile) {
b.setPutSmallFile(b.getPutSmallFileBuilder().setData(data));
}
return b.build();
}
private final ContainerCommandRequestProto header;
private final ByteString data;
private final Supplier<ByteString> contentSupplier
= JavaUtils.memoize(this::buildContent);
private ContainerCommandRequestMessage(
ContainerCommandRequestProto header, ByteString data) {
this.header = Objects.requireNonNull(header, "header == null");
this.data = Objects.requireNonNull(data, "data == null");
}
private ByteString buildContent() {
final ByteString headerBytes = header.toByteString();
return RatisHelper.int2ByteString(headerBytes.size())
.concat(headerBytes)
.concat(data);
}
@Override
public ByteString getContent() {
return contentSupplier.get();
}
@Override
public String toString() {
return header + ", data.size=" + data.size();
}
}

290
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ratis/RatisHelper.java
View File

@ -1,290 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdds.ratis;
import java.io.DataOutputStream;
import java.io.IOException;
import java.security.cert.CertificateException;
import java.security.cert.X509Certificate;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.UUID;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
import org.apache.hadoop.hdds.security.exception.SCMSecurityException;
import org.apache.hadoop.hdds.security.x509.SecurityConfig;
import org.apache.hadoop.hdds.security.x509.certificate.authority.CertificateServer;
import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient;
import org.apache.hadoop.hdds.security.x509.certificate.utils.CertificateCodec;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.ratis.RaftConfigKeys;
import org.apache.ratis.client.RaftClient;
import org.apache.ratis.client.RaftClientConfigKeys;
import org.apache.ratis.conf.RaftProperties;
import org.apache.ratis.grpc.GrpcConfigKeys;
import org.apache.ratis.grpc.GrpcFactory;
import org.apache.ratis.grpc.GrpcTlsConfig;
import org.apache.ratis.proto.RaftProtos;
import org.apache.ratis.protocol.RaftGroup;
import org.apache.ratis.protocol.RaftGroupId;
import org.apache.ratis.protocol.RaftPeer;
import org.apache.ratis.protocol.RaftPeerId;
import org.apache.ratis.retry.RetryPolicies;
import org.apache.ratis.retry.RetryPolicy;
import org.apache.ratis.rpc.RpcType;
import org.apache.ratis.rpc.SupportedRpcType;
import org.apache.ratis.thirdparty.com.google.protobuf.ByteString;
import org.apache.ratis.util.SizeInBytes;
import org.apache.ratis.util.TimeDuration;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Ratis helper methods.
*/
public interface RatisHelper {
Logger LOG = LoggerFactory.getLogger(RatisHelper.class);
static String toRaftPeerIdString(DatanodeDetails id) {
return id.getUuidString();
}
static UUID toDatanodeId(String peerIdString) {
return UUID.fromString(peerIdString);
}
static UUID toDatanodeId(RaftPeerId peerId) {
return toDatanodeId(peerId.toString());
}
static UUID toDatanodeId(RaftProtos.RaftPeerProto peerId) {
return toDatanodeId(RaftPeerId.valueOf(peerId.getId()));
}
static String toRaftPeerAddressString(DatanodeDetails id) {
return id.getIpAddress() + ":" +
id.getPort(DatanodeDetails.Port.Name.RATIS).getValue();
}
static RaftPeerId toRaftPeerId(DatanodeDetails id) {
return RaftPeerId.valueOf(toRaftPeerIdString(id));
}
static RaftPeer toRaftPeer(DatanodeDetails id) {
return new RaftPeer(toRaftPeerId(id), toRaftPeerAddressString(id));
}
static List<RaftPeer> toRaftPeers(Pipeline pipeline) {
return toRaftPeers(pipeline.getNodes());
}
static <E extends DatanodeDetails> List<RaftPeer> toRaftPeers(
List<E> datanodes) {
return datanodes.stream().map(RatisHelper::toRaftPeer)
.collect(Collectors.toList());
}
/* TODO: use a dummy id for all groups for the moment.
* It should be changed to a unique id for each group.
*/
RaftGroupId DUMMY_GROUP_ID =
RaftGroupId.valueOf(ByteString.copyFromUtf8("AOzoneRatisGroup"));
RaftGroup EMPTY_GROUP = RaftGroup.valueOf(DUMMY_GROUP_ID,
Collections.emptyList());
static RaftGroup emptyRaftGroup() {
return EMPTY_GROUP;
}
static RaftGroup newRaftGroup(Collection<RaftPeer> peers) {
return peers.isEmpty()? emptyRaftGroup()
: RaftGroup.valueOf(DUMMY_GROUP_ID, peers);
}
static RaftGroup newRaftGroup(RaftGroupId groupId,
Collection<DatanodeDetails> peers) {
final List<RaftPeer> newPeers = peers.stream()
.map(RatisHelper::toRaftPeer)
.collect(Collectors.toList());
return peers.isEmpty() ? RaftGroup.valueOf(groupId, Collections.emptyList())
: RaftGroup.valueOf(groupId, newPeers);
}
static RaftGroup newRaftGroup(Pipeline pipeline) {
return RaftGroup.valueOf(RaftGroupId.valueOf(pipeline.getId().getId()),
toRaftPeers(pipeline));
}
static RaftClient newRaftClient(RpcType rpcType, Pipeline pipeline,
RetryPolicy retryPolicy, int maxOutStandingRequest,
GrpcTlsConfig tlsConfig, TimeDuration timeout) throws IOException {
return newRaftClient(rpcType, toRaftPeerId(pipeline.getFirstNode()),
newRaftGroup(RaftGroupId.valueOf(pipeline.getId().getId()),
pipeline.getNodes()), retryPolicy, maxOutStandingRequest, tlsConfig,
timeout);
}
static TimeDuration getClientRequestTimeout(Configuration conf) {
// Set the client requestTimeout
final TimeUnit timeUnit =
OzoneConfigKeys.DFS_RATIS_CLIENT_REQUEST_TIMEOUT_DURATION_DEFAULT
.getUnit();
final long duration = conf.getTimeDuration(
OzoneConfigKeys.DFS_RATIS_CLIENT_REQUEST_TIMEOUT_DURATION_KEY,
OzoneConfigKeys.DFS_RATIS_CLIENT_REQUEST_TIMEOUT_DURATION_DEFAULT
.getDuration(), timeUnit);
final TimeDuration clientRequestTimeout =
TimeDuration.valueOf(duration, timeUnit);
return clientRequestTimeout;
}
static RaftClient newRaftClient(RpcType rpcType, RaftPeer leader,
RetryPolicy retryPolicy, int maxOutstandingRequests,
GrpcTlsConfig tlsConfig, TimeDuration clientRequestTimeout) {
return newRaftClient(rpcType, leader.getId(),
newRaftGroup(new ArrayList<>(Arrays.asList(leader))), retryPolicy,
maxOutstandingRequests, tlsConfig, clientRequestTimeout);
}
static RaftClient newRaftClient(RpcType rpcType, RaftPeer leader,
RetryPolicy retryPolicy, int maxOutstandingRequests,
TimeDuration clientRequestTimeout) {
return newRaftClient(rpcType, leader.getId(),
newRaftGroup(new ArrayList<>(Arrays.asList(leader))), retryPolicy,
maxOutstandingRequests, null, clientRequestTimeout);
}
static RaftClient newRaftClient(RpcType rpcType, RaftPeerId leader,
RaftGroup group, RetryPolicy retryPolicy, int maxOutStandingRequest,
GrpcTlsConfig tlsConfig, TimeDuration clientRequestTimeout) {
if (LOG.isTraceEnabled()) {
LOG.trace("newRaftClient: {}, leader={}, group={}",
rpcType, leader, group);
}
final RaftProperties properties = new RaftProperties();
RaftConfigKeys.Rpc.setType(properties, rpcType);
RaftClientConfigKeys.Rpc
.setRequestTimeout(properties, clientRequestTimeout);
GrpcConfigKeys.setMessageSizeMax(properties,
SizeInBytes.valueOf(OzoneConsts.OZONE_SCM_CHUNK_MAX_SIZE));
GrpcConfigKeys.OutputStream.setOutstandingAppendsMax(properties,
maxOutStandingRequest);
RaftClient.Builder builder = RaftClient.newBuilder()
.setRaftGroup(group)
.setLeaderId(leader)
.setProperties(properties)
.setRetryPolicy(retryPolicy);
// TODO: GRPC TLS only for now, netty/hadoop RPC TLS support later.
if (tlsConfig != null && rpcType == SupportedRpcType.GRPC) {
builder.setParameters(GrpcFactory.newRaftParameters(tlsConfig));
}
return builder.build();
}
// For External gRPC client to server with gRPC TLS.
// No mTLS for external client as SCM CA does not issued certificates for them
static GrpcTlsConfig createTlsClientConfig(SecurityConfig conf,
X509Certificate caCert) {
GrpcTlsConfig tlsConfig = null;
if (conf.isSecurityEnabled() && conf.isGrpcTlsEnabled()) {
tlsConfig = new GrpcTlsConfig(null, null,
caCert, false);
}
return tlsConfig;
}
// For Internal gRPC client from SCM to DN with gRPC TLS
static GrpcTlsConfig createTlsClientConfigForSCM(SecurityConfig conf,
CertificateServer certificateServer) throws IOException {
if (conf.isSecurityEnabled() && conf.isGrpcTlsEnabled()) {
try {
X509Certificate caCert =
CertificateCodec.getX509Certificate(
certificateServer.getCACertificate());
return new GrpcTlsConfig(null, null,
caCert, false);
} catch (CertificateException ex) {
throw new SCMSecurityException("Fail to find SCM CA certificate.", ex);
}
}
return null;
}
// For gRPC server running DN container service with gPRC TLS
// No mTLS as the channel is shared for for external client, which
// does not have SCM CA issued certificates.
// In summary:
// authenticate from server to client is via TLS.
// authenticate from client to server is via block token (or container token).
static GrpcTlsConfig createTlsServerConfigForDN(SecurityConfig conf,
CertificateClient caClient) {
if (conf.isSecurityEnabled() && conf.isGrpcTlsEnabled()) {
return new GrpcTlsConfig(
caClient.getPrivateKey(), caClient.getCertificate(),
null, false);
}
return null;
}
static RetryPolicy createRetryPolicy(Configuration conf) {
int maxRetryCount =
conf.getInt(OzoneConfigKeys.DFS_RATIS_CLIENT_REQUEST_MAX_RETRIES_KEY,
OzoneConfigKeys.
DFS_RATIS_CLIENT_REQUEST_MAX_RETRIES_DEFAULT);
long retryInterval = conf.getTimeDuration(OzoneConfigKeys.
DFS_RATIS_CLIENT_REQUEST_RETRY_INTERVAL_KEY, OzoneConfigKeys.
DFS_RATIS_CLIENT_REQUEST_RETRY_INTERVAL_DEFAULT
.toIntExact(TimeUnit.MILLISECONDS), TimeUnit.MILLISECONDS);
TimeDuration sleepDuration =
TimeDuration.valueOf(retryInterval, TimeUnit.MILLISECONDS);
RetryPolicy retryPolicy = RetryPolicies
.retryUpToMaximumCountWithFixedSleep(maxRetryCount, sleepDuration);
return retryPolicy;
}
static Long getMinReplicatedIndex(
Collection<RaftProtos.CommitInfoProto> commitInfos) {
return commitInfos.stream().map(RaftProtos.CommitInfoProto::getCommitIndex)
.min(Long::compareTo).orElse(null);
}
static ByteString int2ByteString(int n) {
final ByteString.Output out = ByteString.newOutput();
try(DataOutputStream dataOut = new DataOutputStream(out)) {
dataOut.writeInt(n);
} catch (IOException e) {
throw new IllegalStateException(
"Failed to write integer n = " + n + " to a ByteString.", e);
}
return out.toByteString();
}
}

22
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ratis/package-info.java
View File

@ -1,22 +0,0 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdds.ratis;
/**
* This package contains classes related to Apache Ratis.
*/

62
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ByteStringConversion.java
View File

@ -1,62 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdds.scm;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.ratis.thirdparty.com.google.protobuf.ByteString;
import org.apache.ratis.thirdparty.com.google.protobuf.UnsafeByteOperations;
import java.nio.ByteBuffer;
import java.util.function.Function;
/**
* Helper class to create a conversion function from ByteBuffer to ByteString
* based on the property
* {@link OzoneConfigKeys#OZONE_UNSAFEBYTEOPERATIONS_ENABLED} in the
* Ozone configuration.
*/
public final class ByteStringConversion {
private ByteStringConversion(){} // no instantiation.
/**
* Creates the conversion function to be used to convert ByteBuffers to
* ByteString instances to be used in protobuf messages.
*
* @param config the Ozone configuration
* @return the conversion function defined by
* {@link OzoneConfigKeys#OZONE_UNSAFEBYTEOPERATIONS_ENABLED}
* @see <pre>ByteBuffer</pre>
*/
public static Function<ByteBuffer, ByteString> createByteBufferConversion(
Configuration config){
boolean unsafeEnabled =
config!=null && config.getBoolean(
OzoneConfigKeys.OZONE_UNSAFEBYTEOPERATIONS_ENABLED,
OzoneConfigKeys.OZONE_UNSAFEBYTEOPERATIONS_ENABLED_DEFAULT);
if (unsafeEnabled) {
return buffer -> UnsafeByteOperations.unsafeWrap(buffer);
} else {
return buffer -> {
ByteString retval = ByteString.copyFrom(buffer);
buffer.flip();
return retval;
};
}
}
}

375
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
View File

@ -1,375 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdds.scm;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.ratis.proto.RaftProtos.ReplicationLevel;
import org.apache.ratis.util.TimeDuration;
import java.util.concurrent.TimeUnit;
/**
* This class contains constants for configuration keys used in SCM.
*/
@InterfaceAudience.Public
@InterfaceStability.Unstable
public final class ScmConfigKeys {
// Location of SCM DB files. For now we just support a single
// metadata dir but in future we may support multiple for redundancy or
// performance.
public static final String OZONE_SCM_DB_DIRS = "ozone.scm.db.dirs";
public static final String DFS_CONTAINER_RATIS_ENABLED_KEY
= "dfs.container.ratis.enabled";
public static final boolean DFS_CONTAINER_RATIS_ENABLED_DEFAULT
= false;
public static final String DFS_CONTAINER_RATIS_RPC_TYPE_KEY
= "dfs.container.ratis.rpc.type";
public static final String DFS_CONTAINER_RATIS_RPC_TYPE_DEFAULT
= "GRPC";
public static final String DFS_CONTAINER_RATIS_NUM_WRITE_CHUNK_THREADS_KEY
= "dfs.container.ratis.num.write.chunk.threads";
public static final int DFS_CONTAINER_RATIS_NUM_WRITE_CHUNK_THREADS_DEFAULT
= 60;
public static final String DFS_CONTAINER_RATIS_REPLICATION_LEVEL_KEY
= "dfs.container.ratis.replication.level";
public static final ReplicationLevel
DFS_CONTAINER_RATIS_REPLICATION_LEVEL_DEFAULT = ReplicationLevel.MAJORITY;
public static final String DFS_CONTAINER_RATIS_NUM_CONTAINER_OP_EXECUTORS_KEY
= "dfs.container.ratis.num.container.op.executors";
public static final int DFS_CONTAINER_RATIS_NUM_CONTAINER_OP_EXECUTORS_DEFAULT
= 10;
public static final String DFS_CONTAINER_RATIS_SEGMENT_SIZE_KEY =
"dfs.container.ratis.segment.size";
public static final String DFS_CONTAINER_RATIS_SEGMENT_SIZE_DEFAULT =
"1MB";
public static final String DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_KEY =
"dfs.container.ratis.segment.preallocated.size";
public static final String
DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_DEFAULT = "16KB";
public static final String
DFS_CONTAINER_RATIS_STATEMACHINEDATA_SYNC_TIMEOUT =
"dfs.container.ratis.statemachinedata.sync.timeout";
public static final TimeDuration
DFS_CONTAINER_RATIS_STATEMACHINEDATA_SYNC_TIMEOUT_DEFAULT =
TimeDuration.valueOf(10, TimeUnit.SECONDS);
public static final String
DFS_CONTAINER_RATIS_STATEMACHINEDATA_SYNC_RETRIES =
"dfs.container.ratis.statemachinedata.sync.retries";
public static final int
DFS_CONTAINER_RATIS_STATEMACHINEDATA_SYNC_RETRIES_DEFAULT = -1;
public static final String
DFS_CONTAINER_RATIS_STATEMACHINE_MAX_PENDING_APPLY_TXNS =
"dfs.container.ratis.statemachine.max.pending.apply-transactions";
// The default value of maximum number of pending state machine apply
// transactions is kept same as default snapshot threshold.
public static final int
DFS_CONTAINER_RATIS_STATEMACHINE_MAX_PENDING_APPLY_TXNS_DEFAULT =
100000;
public static final String DFS_CONTAINER_RATIS_LOG_QUEUE_NUM_ELEMENTS =
"dfs.container.ratis.log.queue.num-elements";
public static final int DFS_CONTAINER_RATIS_LOG_QUEUE_NUM_ELEMENTS_DEFAULT =
1024;
public static final String DFS_CONTAINER_RATIS_LOG_QUEUE_BYTE_LIMIT =
"dfs.container.ratis.log.queue.byte-limit";
public static final String DFS_CONTAINER_RATIS_LOG_QUEUE_BYTE_LIMIT_DEFAULT =
"4GB";
public static final String
DFS_CONTAINER_RATIS_LOG_APPENDER_QUEUE_NUM_ELEMENTS =
"dfs.container.ratis.log.appender.queue.num-elements";
public static final int
DFS_CONTAINER_RATIS_LOG_APPENDER_QUEUE_NUM_ELEMENTS_DEFAULT = 1;
public static final String DFS_CONTAINER_RATIS_LOG_APPENDER_QUEUE_BYTE_LIMIT =
"dfs.container.ratis.log.appender.queue.byte-limit";
public static final String
DFS_CONTAINER_RATIS_LOG_APPENDER_QUEUE_BYTE_LIMIT_DEFAULT = "32MB";
public static final String DFS_CONTAINER_RATIS_LOG_PURGE_GAP =
"dfs.container.ratis.log.purge.gap";
// TODO: Set to 1024 once RATIS issue around purge is fixed.
public static final int DFS_CONTAINER_RATIS_LOG_PURGE_GAP_DEFAULT =
1000000;
public static final String DFS_CONTAINER_RATIS_LEADER_NUM_PENDING_REQUESTS =
"dfs.container.ratis.leader.num.pending.requests";
public static final int
DFS_CONTAINER_RATIS_LEADER_NUM_PENDING_REQUESTS_DEFAULT = 4096;
// expiry interval stateMachineData cache entry inside containerStateMachine
public static final String
DFS_CONTAINER_RATIS_STATEMACHINEDATA_CACHE_EXPIRY_INTERVAL =
"dfs.container.ratis.statemachine.cache.expiry.interval";
public static final String
DFS_CONTAINER_RATIS_STATEMACHINEDATA_CACHE_EXPIRY_INTERVAL_DEFAULT =
"10s";
public static final String DFS_RATIS_CLIENT_REQUEST_TIMEOUT_DURATION_KEY =
"dfs.ratis.client.request.timeout.duration";
public static final TimeDuration
DFS_RATIS_CLIENT_REQUEST_TIMEOUT_DURATION_DEFAULT =
TimeDuration.valueOf(3000, TimeUnit.MILLISECONDS);
public static final String DFS_RATIS_CLIENT_REQUEST_MAX_RETRIES_KEY =
"dfs.ratis.client.request.max.retries";
public static final int DFS_RATIS_CLIENT_REQUEST_MAX_RETRIES_DEFAULT = 180;
public static final String DFS_RATIS_CLIENT_REQUEST_RETRY_INTERVAL_KEY =
"dfs.ratis.client.request.retry.interval";
public static final TimeDuration
DFS_RATIS_CLIENT_REQUEST_RETRY_INTERVAL_DEFAULT =
TimeDuration.valueOf(1000, TimeUnit.MILLISECONDS);
public static final String DFS_RATIS_SERVER_RETRY_CACHE_TIMEOUT_DURATION_KEY =
"dfs.ratis.server.retry-cache.timeout.duration";
public static final TimeDuration
DFS_RATIS_SERVER_RETRY_CACHE_TIMEOUT_DURATION_DEFAULT =
TimeDuration.valueOf(600000, TimeUnit.MILLISECONDS);
public static final String DFS_RATIS_SERVER_REQUEST_TIMEOUT_DURATION_KEY =
"dfs.ratis.server.request.timeout.duration";
public static final TimeDuration
DFS_RATIS_SERVER_REQUEST_TIMEOUT_DURATION_DEFAULT =
TimeDuration.valueOf(3000, TimeUnit.MILLISECONDS);
public static final String
DFS_RATIS_LEADER_ELECTION_MINIMUM_TIMEOUT_DURATION_KEY =
"dfs.ratis.leader.election.minimum.timeout.duration";
public static final TimeDuration
DFS_RATIS_LEADER_ELECTION_MINIMUM_TIMEOUT_DURATION_DEFAULT =
TimeDuration.valueOf(5, TimeUnit.SECONDS);
public static final String DFS_RATIS_SNAPSHOT_THRESHOLD_KEY =
"dfs.ratis.snapshot.threshold";
public static final long DFS_RATIS_SNAPSHOT_THRESHOLD_DEFAULT = 100000;
public static final String DFS_RATIS_SERVER_FAILURE_DURATION_KEY =
"dfs.ratis.server.failure.duration";
public static final TimeDuration
DFS_RATIS_SERVER_FAILURE_DURATION_DEFAULT =
TimeDuration.valueOf(120, TimeUnit.SECONDS);
// TODO : this is copied from OzoneConsts, may need to move to a better place
public static final String OZONE_SCM_CHUNK_SIZE_KEY = "ozone.scm.chunk.size";
// 16 MB by default
public static final String OZONE_SCM_CHUNK_SIZE_DEFAULT = "16MB";
public static final String OZONE_SCM_CLIENT_PORT_KEY =
"ozone.scm.client.port";
public static final int OZONE_SCM_CLIENT_PORT_DEFAULT = 9860;
public static final String OZONE_SCM_DATANODE_PORT_KEY =
"ozone.scm.datanode.port";
public static final int OZONE_SCM_DATANODE_PORT_DEFAULT = 9861;
// OZONE_OM_PORT_DEFAULT = 9862
public static final String OZONE_SCM_BLOCK_CLIENT_PORT_KEY =
"ozone.scm.block.client.port";
public static final int OZONE_SCM_BLOCK_CLIENT_PORT_DEFAULT = 9863;
public static final String OZONE_SCM_SECURITY_SERVICE_PORT_KEY =
"ozone.scm.security.service.port";
public static final int OZONE_SCM_SECURITY_SERVICE_PORT_DEFAULT = 9961;
// Container service client
public static final String OZONE_SCM_CLIENT_ADDRESS_KEY =
"ozone.scm.client.address";
public static final String OZONE_SCM_CLIENT_BIND_HOST_KEY =
"ozone.scm.client.bind.host";
public static final String OZONE_SCM_CLIENT_BIND_HOST_DEFAULT =
"0.0.0.0";
// Block service client
public static final String OZONE_SCM_BLOCK_CLIENT_ADDRESS_KEY =
"ozone.scm.block.client.address";
public static final String OZONE_SCM_BLOCK_CLIENT_BIND_HOST_KEY =
"ozone.scm.block.client.bind.host";
public static final String OZONE_SCM_BLOCK_CLIENT_BIND_HOST_DEFAULT =
"0.0.0.0";
// SCM Security service address.
public static final String OZONE_SCM_SECURITY_SERVICE_ADDRESS_KEY =
"ozone.scm.security.service.address";
public static final String OZONE_SCM_SECURITY_SERVICE_BIND_HOST_KEY =
"ozone.scm.security.service.bind.host";
public static final String OZONE_SCM_SECURITY_SERVICE_BIND_HOST_DEFAULT =
"0.0.0.0";
public static final String OZONE_SCM_DATANODE_ADDRESS_KEY =
"ozone.scm.datanode.address";
public static final String OZONE_SCM_DATANODE_BIND_HOST_KEY =
"ozone.scm.datanode.bind.host";
public static final String OZONE_SCM_DATANODE_BIND_HOST_DEFAULT =
"0.0.0.0";
public static final String OZONE_SCM_HTTP_ENABLED_KEY =
"ozone.scm.http.enabled";
public static final String OZONE_SCM_HTTP_BIND_HOST_KEY =
"ozone.scm.http-bind-host";
public static final String OZONE_SCM_HTTPS_BIND_HOST_KEY =
"ozone.scm.https-bind-host";
public static final String OZONE_SCM_HTTP_ADDRESS_KEY =
"ozone.scm.http-address";
public static final String OZONE_SCM_HTTPS_ADDRESS_KEY =
"ozone.scm.https-address";
public static final String HDDS_SCM_KERBEROS_KEYTAB_FILE_KEY =
"hdds.scm.kerberos.keytab.file";
public static final String HDDS_SCM_KERBEROS_PRINCIPAL_KEY =
"hdds.scm.kerberos.principal";
public static final String OZONE_SCM_HTTP_BIND_HOST_DEFAULT = "0.0.0.0";
public static final int OZONE_SCM_HTTP_BIND_PORT_DEFAULT = 9876;
public static final int OZONE_SCM_HTTPS_BIND_PORT_DEFAULT = 9877;
public static final String HDDS_REST_HTTP_ADDRESS_KEY =
"hdds.rest.http-address";
public static final String HDDS_REST_HTTP_ADDRESS_DEFAULT = "0.0.0.0:9880";
public static final String HDDS_DATANODE_DIR_KEY = "hdds.datanode.dir";
public static final String HDDS_REST_CSRF_ENABLED_KEY =
"hdds.rest.rest-csrf.enabled";
public static final boolean HDDS_REST_CSRF_ENABLED_DEFAULT = false;
public static final String HDDS_REST_NETTY_HIGH_WATERMARK =
"hdds.rest.netty.high.watermark";
public static final int HDDS_REST_NETTY_HIGH_WATERMARK_DEFAULT = 65536;
public static final int HDDS_REST_NETTY_LOW_WATERMARK_DEFAULT = 32768;
public static final String HDDS_REST_NETTY_LOW_WATERMARK =
"hdds.rest.netty.low.watermark";
public static final String OZONE_SCM_HANDLER_COUNT_KEY =
"ozone.scm.handler.count.key";
public static final int OZONE_SCM_HANDLER_COUNT_DEFAULT = 10;
public static final String OZONE_SCM_SECURITY_HANDLER_COUNT_KEY =
"ozone.scm.security.handler.count.key";
public static final int OZONE_SCM_SECURITY_HANDLER_COUNT_DEFAULT = 2;
public static final String OZONE_SCM_DEADNODE_INTERVAL =
"ozone.scm.dead.node.interval";
public static final String OZONE_SCM_DEADNODE_INTERVAL_DEFAULT =
"10m";
public static final String OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL =
"ozone.scm.heartbeat.thread.interval";
public static final String OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL_DEFAULT =
"3s";
public static final String OZONE_SCM_STALENODE_INTERVAL =
"ozone.scm.stale.node.interval";
public static final String OZONE_SCM_STALENODE_INTERVAL_DEFAULT =
"5m";
public static final String OZONE_SCM_HEARTBEAT_RPC_TIMEOUT =
"ozone.scm.heartbeat.rpc-timeout";
public static final String OZONE_SCM_HEARTBEAT_RPC_TIMEOUT_DEFAULT =
"1s";
/**
* Defines how frequently we will log the missing of heartbeat to a specific
* SCM. In the default case we will write a warning message for each 10
* sequential heart beats that we miss to a specific SCM. This is to avoid
* overrunning the log with lots of HB missed Log statements.
*/
public static final String OZONE_SCM_HEARTBEAT_LOG_WARN_INTERVAL_COUNT =
"ozone.scm.heartbeat.log.warn.interval.count";
public static final int OZONE_SCM_HEARTBEAT_LOG_WARN_DEFAULT =
10;
// ozone.scm.names key is a set of DNS | DNS:PORT | IP Address | IP:PORT.
// Written as a comma separated string. e.g. scm1, scm2:8020, 7.7.7.7:7777
//
// If this key is not specified datanodes will not be able to find
// SCM. The SCM membership can be dynamic, so this key should contain
// all possible SCM names. Once the SCM leader is discovered datanodes will
// get the right list of SCMs to heartbeat to from the leader.
// While it is good for the datanodes to know the names of all SCM nodes,
// it is sufficient to actually know the name of on working SCM. That SCM
// will be able to return the information about other SCMs that are part of
// the SCM replicated Log.
//
//In case of a membership change, any one of the SCM machines will be
// able to send back a new list to the datanodes.
public static final String OZONE_SCM_NAMES = "ozone.scm.names";
public static final int OZONE_SCM_DEFAULT_PORT =
OZONE_SCM_DATANODE_PORT_DEFAULT;
// The path where datanode ID is to be written to.
// if this value is not set then container startup will fail.
public static final String OZONE_SCM_DATANODE_ID_DIR =
"ozone.scm.datanode.id.dir";
public static final String OZONE_SCM_DB_CACHE_SIZE_MB =
"ozone.scm.db.cache.size.mb";
public static final int OZONE_SCM_DB_CACHE_SIZE_DEFAULT = 128;
public static final String OZONE_SCM_CONTAINER_SIZE =
"ozone.scm.container.size";
public static final String OZONE_SCM_CONTAINER_SIZE_DEFAULT = "5GB";
public static final String OZONE_SCM_CONTAINER_PLACEMENT_IMPL_KEY =
"ozone.scm.container.placement.impl";
public static final String OZONE_SCM_PIPELINE_OWNER_CONTAINER_COUNT =
"ozone.scm.pipeline.owner.container.count";
public static final int OZONE_SCM_PIPELINE_OWNER_CONTAINER_COUNT_DEFAULT = 3;
public static final String
OZONE_SCM_KEY_VALUE_CONTAINER_DELETION_CHOOSING_POLICY =
"ozone.scm.keyvalue.container.deletion-choosing.policy";
public static final String OZONE_SCM_CONTAINER_CREATION_LEASE_TIMEOUT =
"ozone.scm.container.creation.lease.timeout";
public static final String
OZONE_SCM_CONTAINER_CREATION_LEASE_TIMEOUT_DEFAULT = "60s";
public static final String OZONE_SCM_PIPELINE_DESTROY_TIMEOUT =
"ozone.scm.pipeline.destroy.timeout";
public static final String OZONE_SCM_PIPELINE_DESTROY_TIMEOUT_DEFAULT =
"66s";
public static final String OZONE_SCM_PIPELINE_CREATION_INTERVAL =
"ozone.scm.pipeline.creation.interval";
public static final String OZONE_SCM_PIPELINE_CREATION_INTERVAL_DEFAULT =
"120s";
public static final String OZONE_SCM_BLOCK_DELETION_MAX_RETRY =
"ozone.scm.block.deletion.max.retry";
public static final int OZONE_SCM_BLOCK_DELETION_MAX_RETRY_DEFAULT = 4096;
public static final String HDDS_SCM_WATCHER_TIMEOUT =
"hdds.scm.watcher.timeout";
public static final String HDDS_SCM_WATCHER_TIMEOUT_DEFAULT =
"10m";
public static final String
HDDS_SCM_HTTP_KERBEROS_PRINCIPAL_KEY =
"hdds.scm.http.kerberos.principal";
public static final String
HDDS_SCM_HTTP_KERBEROS_KEYTAB_FILE_KEY =
"hdds.scm.http.kerberos.keytab";
// Network topology
public static final String OZONE_SCM_NETWORK_TOPOLOGY_SCHEMA_FILE =
"ozone.scm.network.topology.schema.file";
public static final String OZONE_SCM_NETWORK_TOPOLOGY_SCHEMA_FILE_DEFAULT =
"network-topology-default.xml";
public static final String HDDS_TRACING_ENABLED = "hdds.tracing.enabled";
public static final boolean HDDS_TRACING_ENABLED_DEFAULT = true;
/**
* Never constructed.
*/
private ScmConfigKeys() {
}
}

81
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmInfo.java
View File

@ -1,81 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdds.scm;
/**
* ScmInfo wraps the result returned from SCM#getScmInfo which
* contains clusterId and the SCM Id.
*/
public final class ScmInfo {
private String clusterId;
private String scmId;
/**
* Builder for ScmInfo.
*/
public static class Builder {
private String clusterId;
private String scmId;
/**
* sets the cluster id.
* @param cid clusterId to be set
* @return Builder for ScmInfo
*/
public Builder setClusterId(String cid) {
this.clusterId = cid;
return this;
}
/**
* sets the scmId.
* @param id scmId
* @return Builder for scmInfo
*/
public Builder setScmId(String id) {
this.scmId = id;
return this;
}
public ScmInfo build() {
return new ScmInfo(clusterId, scmId);
}
}
private ScmInfo(String clusterId, String scmId) {
this.clusterId = clusterId;
this.scmId = scmId;
}
/**
* Gets the clusterId from the Version file.
* @return ClusterId
*/
public String getClusterId() {
return clusterId;
}
/**
* Gets the SCM Id from the Version file.
* @return SCM Id
*/
public String getScmId() {
return scmId;
}
}

82
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientReply.java
View File

@ -1,82 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdds.scm;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
.ContainerCommandResponseProto;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.CompletableFuture;
/**
* This class represents the reply from XceiverClient.
*/
public class XceiverClientReply {
private CompletableFuture<ContainerCommandResponseProto> response;
private Long logIndex;
/**
* List of datanodes where the command got executed and reply is received.
* If there is an exception in the reply, these datanodes will inform
* about the servers where there is a failure.
*/
private List<DatanodeDetails> datanodes;
public XceiverClientReply(
CompletableFuture<ContainerCommandResponseProto> response) {
this(response, null);
}
public XceiverClientReply(
CompletableFuture<ContainerCommandResponseProto> response,
List<DatanodeDetails> datanodes) {
this.logIndex = (long) 0;
this.response = response;
this.datanodes = datanodes == null ? new ArrayList<>() : datanodes;
}
public CompletableFuture<ContainerCommandResponseProto> getResponse() {
return response;
}
public long getLogIndex() {
return logIndex;
}
public void setLogIndex(Long logIndex) {
this.logIndex = logIndex;
}
public List<DatanodeDetails> getDatanodes() {
return datanodes;
}
public void addDatanode(DatanodeDetails dn) {
datanodes.add(dn);
}
public void setResponse(
CompletableFuture<ContainerCommandResponseProto> response) {
this.response = response;
}
}

179
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientSpi.java
View File

@ -1,179 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdds.scm;
import java.io.Closeable;
import java.io.IOException;
import java.util.List;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandRequestProto;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandResponseProto;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.hdds.scm.storage.CheckedBiFunction;
/**
* A Client for the storageContainer protocol.
*/
public abstract class XceiverClientSpi implements Closeable {
final private AtomicInteger referenceCount;
private boolean isEvicted;
XceiverClientSpi() {
this.referenceCount = new AtomicInteger(0);
this.isEvicted = false;
}
void incrementReference() {
this.referenceCount.incrementAndGet();
}
void decrementReference() {
this.referenceCount.decrementAndGet();
cleanup();
}
void setEvicted() {
isEvicted = true;
cleanup();
}
// close the xceiverClient only if,
// 1) there is no refcount on the client
// 2) it has been evicted from the cache.
private void cleanup() {
if (referenceCount.get() == 0 && isEvicted) {
close();
}
}
@VisibleForTesting
public int getRefcount() {
return referenceCount.get();
}
/**
* Connects to the leader in the pipeline.
*/
public abstract void connect() throws Exception;
/**
* Connects to the leader in the pipeline using encoded token. To be used
* in a secure cluster.
*/
public abstract void connect(String encodedToken) throws Exception;
@Override
public abstract void close();
/**
* Returns the pipeline of machines that host the container used by this
* client.
*
* @return pipeline of machines that host the container
*/
public abstract Pipeline getPipeline();
/**
* Sends a given command to server and gets the reply back.
* @param request Request
* @return Response to the command
* @throws IOException
*/
public ContainerCommandResponseProto sendCommand(
ContainerCommandRequestProto request) throws IOException {
try {
XceiverClientReply reply;
reply = sendCommandAsync(request);
ContainerCommandResponseProto responseProto = reply.getResponse().get();
return responseProto;
} catch (ExecutionException | InterruptedException e) {
throw new IOException("Failed to command " + request, e);
}
}
/**
* Sends a given command to server and gets the reply back along with
* the server associated info.
* @param request Request
* @param validators functions to validate the response
* @return Response to the command
* @throws IOException
*/
public ContainerCommandResponseProto sendCommand(
ContainerCommandRequestProto request, List<CheckedBiFunction> validators)
throws IOException {
try {
XceiverClientReply reply;
reply = sendCommandAsync(request);
ContainerCommandResponseProto responseProto = reply.getResponse().get();
for (CheckedBiFunction function : validators) {
function.apply(request, responseProto);
}
return responseProto;
} catch (ExecutionException | InterruptedException e) {
throw new IOException("Failed to command " + request, e);
}
}
/**
* Sends a given command to server gets a waitable future back.
*
* @param request Request
* @return Response to the command
* @throws IOException
*/
public abstract XceiverClientReply
sendCommandAsync(ContainerCommandRequestProto request)
throws IOException, ExecutionException, InterruptedException;
/**
* Returns pipeline Type.
*
* @return - {Stand_Alone, Ratis or Chained}
*/
public abstract HddsProtos.ReplicationType getPipelineType();
/**
* Check if an specfic commitIndex is replicated to majority/all servers.
* @param index index to watch for
* @param timeout timeout provided for the watch operation to complete
* @return reply containing the min commit index replicated to all or majority
* servers in case of a failure
* @throws InterruptedException
* @throws ExecutionException
* @throws TimeoutException
* @throws IOException
*/
public abstract XceiverClientReply watchForCommit(long index, long timeout)
throws InterruptedException, ExecutionException, TimeoutException,
IOException;
/**
* returns the min commit index replicated to all servers.
* @return min commit index replicated to all servers.
*/
public abstract long getReplicatedMinCommitIndex();
}

241
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/client/ScmClient.java
View File

@ -1,241 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdds.scm.client;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline;
import org.apache.hadoop.hdds.scm.container.ContainerInfo;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
.ContainerDataProto;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import java.io.Closeable;
import java.io.IOException;
import java.util.List;
/**
* The interface to call into underlying container layer.
*
* Written as interface to allow easy testing: implement a mock container layer
* for standalone testing of CBlock API without actually calling into remote
* containers. Actual container layer can simply re-implement this.
*
* NOTE this is temporarily needed class. When SCM containers are full-fledged,
* this interface will likely be removed.
*/
@InterfaceStability.Unstable
public interface ScmClient extends Closeable {
/**
* Creates a Container on SCM and returns the pipeline.
* @return ContainerInfo
* @throws IOException
*/
ContainerWithPipeline createContainer(String owner) throws IOException;
/**
* Gets a container by Name -- Throws if the container does not exist.
* @param containerId - Container ID
* @return Pipeline
* @throws IOException
*/
ContainerInfo getContainer(long containerId) throws IOException;
/**
* Gets a container by Name -- Throws if the container does not exist.
* @param containerId - Container ID
* @return ContainerWithPipeline
* @throws IOException
*/
ContainerWithPipeline getContainerWithPipeline(long containerId)
throws IOException;
/**
* Close a container.
*
* @param containerId - ID of the container.
* @param pipeline - Pipeline where the container is located.
* @throws IOException
*/
void closeContainer(long containerId, Pipeline pipeline) throws IOException;
/**
* Close a container.
*
* @param containerId - ID of the container.
* @throws IOException
*/
void closeContainer(long containerId) throws IOException;
/**
* Deletes an existing container.
* @param containerId - ID of the container.
* @param pipeline - Pipeline that represents the container.
* @param force - true to forcibly delete the container.
* @throws IOException
*/
void deleteContainer(long containerId, Pipeline pipeline, boolean force)
throws IOException;
/**
* Deletes an existing container.
* @param containerId - ID of the container.
* @param force - true to forcibly delete the container.
* @throws IOException
*/
void deleteContainer(long containerId, boolean force) throws IOException;
/**
* Lists a range of containers and get their info.
*
* @param startContainerID start containerID.
* @param count count must be {@literal >} 0.
*
* @return a list of pipeline.
* @throws IOException
*/
List<ContainerInfo> listContainer(long startContainerID,
int count) throws IOException;
/**
* Read meta data from an existing container.
* @param containerID - ID of the container.
* @param pipeline - Pipeline where the container is located.
* @return ContainerInfo
* @throws IOException
*/
ContainerDataProto readContainer(long containerID, Pipeline pipeline)
throws IOException;
/**
* Read meta data from an existing container.
* @param containerID - ID of the container.
* @return ContainerInfo
* @throws IOException
*/
ContainerDataProto readContainer(long containerID)
throws IOException;
/**
* Gets the container size -- Computed by SCM from Container Reports.
* @param containerID - ID of the container.
* @return number of bytes used by this container.
* @throws IOException
*/
long getContainerSize(long containerID) throws IOException;
/**
* Creates a Container on SCM and returns the pipeline.
* @param type - Replication Type.
* @param replicationFactor - Replication Factor
* @return ContainerInfo
* @throws IOException - in case of error.
*/
ContainerWithPipeline createContainer(HddsProtos.ReplicationType type,
HddsProtos.ReplicationFactor replicationFactor,
String owner) throws IOException;
/**
* Returns a set of Nodes that meet a query criteria.
* @param nodeStatuses - Criteria that we want the node to have.
* @param queryScope - Query scope - Cluster or pool.
* @param poolName - if it is pool, a pool name is required.
* @return A set of nodes that meet the requested criteria.
* @throws IOException
*/
List<HddsProtos.Node> queryNode(HddsProtos.NodeState nodeStatuses,
HddsProtos.QueryScope queryScope, String poolName) throws IOException;
/**
* Creates a specified replication pipeline.
* @param type - Type
* @param factor - Replication factor
* @param nodePool - Set of machines.
* @throws IOException
*/
Pipeline createReplicationPipeline(HddsProtos.ReplicationType type,
HddsProtos.ReplicationFactor factor, HddsProtos.NodePool nodePool)
throws IOException;
/**
* Returns the list of active Pipelines.
*
* @return list of Pipeline
* @throws IOException in case of any exception
*/
List<Pipeline> listPipelines() throws IOException;
/**
* Activates the pipeline given a pipeline ID.
*
* @param pipelineID PipelineID to activate.
* @throws IOException In case of exception while activating the pipeline
*/
void activatePipeline(HddsProtos.PipelineID pipelineID) throws IOException;
/**
* Deactivates the pipeline given a pipeline ID.
*
* @param pipelineID PipelineID to deactivate.
* @throws IOException In case of exception while deactivating the pipeline
*/
void deactivatePipeline(HddsProtos.PipelineID pipelineID) throws IOException;
/**
* Closes the pipeline given a pipeline ID.
*
* @param pipelineID PipelineID to close.
* @throws IOException In case of exception while closing the pipeline
*/
void closePipeline(HddsProtos.PipelineID pipelineID) throws IOException;
/**
* Check if SCM is in safe mode.
*
* @return Returns true if SCM is in safe mode else returns false.
* @throws IOException
*/
boolean inSafeMode() throws IOException;
/**
* Force SCM out of safe mode.
*
* @return returns true if operation is successful.
* @throws IOException
*/
boolean forceExitSafeMode() throws IOException;
/**
* Start ReplicationManager.
*/
void startReplicationManager() throws IOException;
/**
* Stop ReplicationManager.
*/
void stopReplicationManager() throws IOException;
/**
* Returns ReplicationManager status.
*
* @return True if ReplicationManager is running, false otherwise.
*/
boolean getReplicationManagerStatus() throws IOException;
}

24
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/client/package-info.java
View File

@ -1,24 +0,0 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdds.scm.client;
/**
* This package contains classes for the client of the storage container
* protocol.
*/

46
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerException.java
View File

@ -1,46 +0,0 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations under
* the License.
*/
package org.apache.hadoop.hdds.scm.container;
import java.io.IOException;
/**
* Signals that ContainerException of some sort has occurred. This is parent
* of all the exceptions thrown by ContainerManager.
*/
public class ContainerException extends IOException {
/**
* Constructs an {@code ContainerException} with {@code null}
* as its error detail message.
*/
public ContainerException() {
super();
}
/**
* Constructs an {@code ContainerException} with the specified detail message.
*
* @param message
* The detail message (which is saved for later retrieval
* by the {@link #getMessage()} method)
*/
public ContainerException(String message) {
super(message);
}
}

107
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerID.java
View File

@ -1,107 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations under
* the License.
*
*/
package org.apache.hadoop.hdds.scm.container;
import com.google.common.base.Preconditions;
import com.google.common.primitives.Longs;
import org.apache.commons.lang3.builder.CompareToBuilder;
import org.apache.commons.lang3.builder.EqualsBuilder;
import org.apache.commons.lang3.builder.HashCodeBuilder;
/**
* Container ID is an integer that is a value between 1..MAX_CONTAINER ID.
* <p>
* We are creating a specific type for this to avoid mixing this with
* normal integers in code.
*/
public final class ContainerID implements Comparable<ContainerID> {
private final long id;
// TODO: make this private.
/**
* Constructs ContainerID.
*
* @param id int
*/
public ContainerID(long id) {
this.id = id;
}
/**
* Factory method for creation of ContainerID.
* @param containerID long
* @return ContainerID.
*/
public static ContainerID valueof(final long containerID) {
Preconditions.checkState(containerID > 0,
"Container ID should be a positive long. "+ containerID);
return new ContainerID(containerID);
}
/**
* Returns int representation of ID.
*
* @return int
*/
public long getId() {
return id;
}
public byte[] getBytes() {
return Longs.toByteArray(id);
}
@Override
public boolean equals(final Object o) {
if (this == o) {
return true;
}
if (o == null || getClass() != o.getClass()) {
return false;
}
final ContainerID that = (ContainerID) o;
return new EqualsBuilder()
.append(getId(), that.getId())
.isEquals();
}
@Override
public int hashCode() {
return new HashCodeBuilder(61, 71)
.append(getId())
.toHashCode();
}
@Override
public int compareTo(final ContainerID that) {
Preconditions.checkNotNull(that);
return new CompareToBuilder()
.append(this.getId(), that.getId())
.build();
}
@Override
public String toString() {
return "#" + id;
}
}

471
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerInfo.java
View File

@ -1,471 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdds.scm.container;
import static java.lang.Math.max;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.PropertyAccessor;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.ObjectWriter;
import com.google.common.base.Preconditions;
import java.io.Externalizable;
import java.io.IOException;
import java.io.ObjectInput;
import java.io.ObjectOutput;
import java.util.Arrays;
import java.util.Comparator;
import org.apache.commons.lang3.builder.EqualsBuilder;
import org.apache.commons.lang3.builder.HashCodeBuilder;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType;
import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
import org.apache.hadoop.util.Time;
/**
* Class wraps ozone container info.
*/
public class ContainerInfo implements Comparator<ContainerInfo>,
Comparable<ContainerInfo>, Externalizable {
private static final ObjectWriter WRITER;
private static final String SERIALIZATION_ERROR_MSG = "Java serialization not"
+ " supported. Use protobuf instead.";
static {
ObjectMapper mapper = new ObjectMapper();
mapper.setVisibility(PropertyAccessor.FIELD, JsonAutoDetect.Visibility.ANY);
mapper
.setVisibility(PropertyAccessor.GETTER, JsonAutoDetect.Visibility.NONE);
WRITER = mapper.writerWithDefaultPrettyPrinter();
}
private HddsProtos.LifeCycleState state;
@JsonIgnore
private PipelineID pipelineID;
private ReplicationFactor replicationFactor;
private ReplicationType replicationType;
private long usedBytes;
private long numberOfKeys;
private long lastUsed;
// The wall-clock ms since the epoch at which the current state enters.
private long stateEnterTime;
private String owner;
private long containerID;
private long deleteTransactionId;
// The sequenceId of a close container cannot change, and all the
// container replica should have the same sequenceId.
private long sequenceId;
/**
* Allows you to maintain private data on ContainerInfo. This is not
* serialized via protobuf, just allows us to maintain some private data.
*/
@JsonIgnore
private byte[] data;
@SuppressWarnings("parameternumber")
ContainerInfo(
long containerID,
HddsProtos.LifeCycleState state,
PipelineID pipelineID,
long usedBytes,
long numberOfKeys,
long stateEnterTime,
String owner,
long deleteTransactionId,
long sequenceId,
ReplicationFactor replicationFactor,
ReplicationType repType) {
this.containerID = containerID;
this.pipelineID = pipelineID;
this.usedBytes = usedBytes;
this.numberOfKeys = numberOfKeys;
this.lastUsed = Time.monotonicNow();
this.state = state;
this.stateEnterTime = stateEnterTime;
this.owner = owner;
this.deleteTransactionId = deleteTransactionId;
this.sequenceId = sequenceId;
this.replicationFactor = replicationFactor;
this.replicationType = repType;
}
/**
* Needed for serialization findbugs.
*/
public ContainerInfo() {
}
public static ContainerInfo fromProtobuf(HddsProtos.ContainerInfoProto info) {
ContainerInfo.Builder builder = new ContainerInfo.Builder();
return builder.setPipelineID(
PipelineID.getFromProtobuf(info.getPipelineID()))
.setUsedBytes(info.getUsedBytes())
.setNumberOfKeys(info.getNumberOfKeys())
.setState(info.getState())
.setStateEnterTime(info.getStateEnterTime())
.setOwner(info.getOwner())
.setContainerID(info.getContainerID())
.setDeleteTransactionId(info.getDeleteTransactionId())
.setReplicationFactor(info.getReplicationFactor())
.setReplicationType(info.getReplicationType())
.build();
}
public long getContainerID() {
return containerID;
}
public HddsProtos.LifeCycleState getState() {
return state;
}
public void setState(HddsProtos.LifeCycleState state) {
this.state = state;
}
public long getStateEnterTime() {
return stateEnterTime;
}
public ReplicationFactor getReplicationFactor() {
return replicationFactor;
}
public PipelineID getPipelineID() {
return pipelineID;
}
public long getUsedBytes() {
return usedBytes;
}
public void setUsedBytes(long value) {
usedBytes = value;
}
public long getNumberOfKeys() {
return numberOfKeys;
}
public void setNumberOfKeys(long value) {
numberOfKeys = value;
}
public long getDeleteTransactionId() {
return deleteTransactionId;
}
public long getSequenceId() {
return sequenceId;
}
public void updateDeleteTransactionId(long transactionId) {
deleteTransactionId = max(transactionId, deleteTransactionId);
}
public void updateSequenceId(long sequenceID) {
assert (isOpen() || state == HddsProtos.LifeCycleState.QUASI_CLOSED);
sequenceId = max(sequenceID, sequenceId);
}
public ContainerID containerID() {
return new ContainerID(getContainerID());
}
/**
* Gets the last used time from SCM's perspective.
*
* @return time in milliseconds.
*/
public long getLastUsed() {
return lastUsed;
}
public ReplicationType getReplicationType() {
return replicationType;
}
public void updateLastUsedTime() {
lastUsed = Time.monotonicNow();
}
public HddsProtos.ContainerInfoProto getProtobuf() {
HddsProtos.ContainerInfoProto.Builder builder =
HddsProtos.ContainerInfoProto.newBuilder();
Preconditions.checkState(containerID > 0);
return builder.setContainerID(getContainerID())
.setUsedBytes(getUsedBytes())
.setNumberOfKeys(getNumberOfKeys()).setState(getState())
.setStateEnterTime(getStateEnterTime()).setContainerID(getContainerID())
.setDeleteTransactionId(getDeleteTransactionId())
.setPipelineID(getPipelineID().getProtobuf())
.setReplicationFactor(getReplicationFactor())
.setReplicationType(getReplicationType())
.setOwner(getOwner())
.build();
}
public String getOwner() {
return owner;
}
public void setOwner(String owner) {
this.owner = owner;
}
@Override
public String toString() {
return "ContainerInfo{"
+ "id=" + containerID
+ ", state=" + state
+ ", pipelineID=" + pipelineID
+ ", stateEnterTime=" + stateEnterTime
+ ", owner=" + owner
+ '}';
}
@Override
public boolean equals(Object o) {
if (this == o) {
return true;
}
if (o == null || getClass() != o.getClass()) {
return false;
}
ContainerInfo that = (ContainerInfo) o;
return new EqualsBuilder()
.append(getContainerID(), that.getContainerID())
// TODO : Fix this later. If we add these factors some tests fail.
// So Commenting this to continue and will enforce this with
// Changes in pipeline where we remove Container Name to
// SCMContainerinfo from Pipeline.
// .append(pipeline.getFactor(), that.pipeline.getFactor())
// .append(pipeline.getType(), that.pipeline.getType())
.append(owner, that.owner)
.isEquals();
}
@Override
public int hashCode() {
return new HashCodeBuilder(11, 811)
.append(getContainerID())
.append(getOwner())
.toHashCode();
}
/**
* Compares its two arguments for order. Returns a negative integer, zero, or
* a positive integer as the first argument is less than, equal to, or greater
* than the second.<p>
*
* @param o1 the first object to be compared.
* @param o2 the second object to be compared.
* @return a negative integer, zero, or a positive integer as the first
* argument is less than, equal to, or greater than the second.
* @throws NullPointerException if an argument is null and this comparator
* does not permit null arguments
* @throws ClassCastException if the arguments' types prevent them from
* being compared by this comparator.
*/
@Override
public int compare(ContainerInfo o1, ContainerInfo o2) {
return Long.compare(o1.getLastUsed(), o2.getLastUsed());
}
/**
* Compares this object with the specified object for order. Returns a
* negative integer, zero, or a positive integer as this object is less than,
* equal to, or greater than the specified object.
*
* @param o the object to be compared.
* @return a negative integer, zero, or a positive integer as this object is
* less than, equal to, or greater than the specified object.
* @throws NullPointerException if the specified object is null
* @throws ClassCastException if the specified object's type prevents it
* from being compared to this object.
*/
@Override
public int compareTo(ContainerInfo o) {
return this.compare(this, o);
}
/**
* Returns a JSON string of this object.
*
* @return String - json string
* @throws IOException
*/
public String toJsonString() throws IOException {
return WRITER.writeValueAsString(this);
}
/**
* Returns private data that is set on this containerInfo.
*
* @return blob, the user can interpret it any way they like.
*/
public byte[] getData() {
if (this.data != null) {
return Arrays.copyOf(this.data, this.data.length);
} else {
return null;
}
}
/**
* Set private data on ContainerInfo object.
*
* @param data -- private data.
*/
public void setData(byte[] data) {
if (data != null) {
this.data = Arrays.copyOf(data, data.length);
}
}
/**
* Throws IOException as default java serialization is not supported. Use
* serialization via protobuf instead.
*
* @param out the stream to write the object to
* @throws IOException Includes any I/O exceptions that may occur
* @serialData Overriding methods should use this tag to describe
* the data layout of this Externalizable object.
* List the sequence of element types and, if possible,
* relate the element to a public/protected field and/or
* method of this Externalizable class.
*/
@Override
public void writeExternal(ObjectOutput out) throws IOException {
throw new IOException(SERIALIZATION_ERROR_MSG);
}
/**
* Throws IOException as default java serialization is not supported. Use
* serialization via protobuf instead.
*
* @param in the stream to read data from in order to restore the object
* @throws IOException if I/O errors occur
* @throws ClassNotFoundException If the class for an object being
* restored cannot be found.
*/
@Override
public void readExternal(ObjectInput in)
throws IOException, ClassNotFoundException {
throw new IOException(SERIALIZATION_ERROR_MSG);
}
/**
* Builder class for ContainerInfo.
*/
public static class Builder {
private HddsProtos.LifeCycleState state;
private long used;
private long keys;
private long stateEnterTime;
private String owner;
private long containerID;
private long deleteTransactionId;
private long sequenceId;
private PipelineID pipelineID;
private ReplicationFactor replicationFactor;
private ReplicationType replicationType;
public Builder setReplicationType(
ReplicationType repType) {
this.replicationType = repType;
return this;
}
public Builder setPipelineID(PipelineID pipelineId) {
this.pipelineID = pipelineId;
return this;
}
public Builder setReplicationFactor(ReplicationFactor repFactor) {
this.replicationFactor = repFactor;
return this;
}
public Builder setContainerID(long id) {
Preconditions.checkState(id >= 0);
this.containerID = id;
return this;
}
public Builder setState(HddsProtos.LifeCycleState lifeCycleState) {
this.state = lifeCycleState;
return this;
}
public Builder setUsedBytes(long bytesUsed) {
this.used = bytesUsed;
return this;
}
public Builder setNumberOfKeys(long keyCount) {
this.keys = keyCount;
return this;
}
public Builder setStateEnterTime(long time) {
this.stateEnterTime = time;
return this;
}
public Builder setOwner(String containerOwner) {
this.owner = containerOwner;
return this;
}
public Builder setDeleteTransactionId(long deleteTransactionID) {
this.deleteTransactionId = deleteTransactionID;
return this;
}
public Builder setSequenceId(long sequenceID) {
this.sequenceId = sequenceID;
return this;
}
public ContainerInfo build() {
return new ContainerInfo(containerID, state, pipelineID,
used, keys, stateEnterTime, owner, deleteTransactionId,
sequenceId, replicationFactor, replicationType);
}
}
/**
* Check if a container is in open state, this will check if the
* container is either open or closing state. Any containers in these states
* is managed as an open container by SCM.
*/
public boolean isOpen() {
return state == HddsProtos.LifeCycleState.OPEN
|| state == HddsProtos.LifeCycleState.CLOSING;
}
}

44
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerNotFoundException.java
View File

@ -1,44 +0,0 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations under
* the License.
*/
package org.apache.hadoop.hdds.scm.container;
/**
* Signals that a container is missing from ContainerManager.
*/
public class ContainerNotFoundException extends ContainerException {
/**
* Constructs an {@code ContainerNotFoundException} with {@code null}
* as its error detail message.
*/
public ContainerNotFoundException() {
super();
}
/**
* Constructs an {@code ContainerNotFoundException} with the specified
* detail message.
*
* @param message
* The detail message (which is saved for later retrieval
* by the {@link #getMessage()} method)
*/
public ContainerNotFoundException(String message) {
super(message);
}
}

45
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerReplicaNotFoundException.java
View File

@ -1,45 +0,0 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations under
* the License.
*/
package org.apache.hadoop.hdds.scm.container;
/**
* Signals that a ContainerReplica is missing from the Container in
* ContainerManager.
*/
public class ContainerReplicaNotFoundException extends ContainerException {
/**
* Constructs an {@code ContainerReplicaNotFoundException} with {@code null}
* as its error detail message.
*/
public ContainerReplicaNotFoundException() {
super();
}
/**
* Constructs an {@code ContainerReplicaNotFoundException} with the
* specified detail message.
*
* @param message
* The detail message (which is saved for later retrieval
* by the {@link #getMessage()} method)
*/
public ContainerReplicaNotFoundException(String message) {
super(message);
}
}

66
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/AllocatedBlock.java
View File

@ -1,66 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdds.scm.container.common.helpers;
import org.apache.hadoop.hdds.client.ContainerBlockID;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
/**
* Allocated block wraps the result returned from SCM#allocateBlock which
* contains a Pipeline and the key.
*/
public final class AllocatedBlock {
private Pipeline pipeline;
private ContainerBlockID containerBlockID;
/**
* Builder for AllocatedBlock.
*/
public static class Builder {
private Pipeline pipeline;
private ContainerBlockID containerBlockID;
public Builder setPipeline(Pipeline p) {
this.pipeline = p;
return this;
}
public Builder setContainerBlockID(ContainerBlockID blockId) {
this.containerBlockID = blockId;
return this;
}
public AllocatedBlock build() {
return new AllocatedBlock(pipeline, containerBlockID);
}
}
private AllocatedBlock(Pipeline pipeline, ContainerBlockID containerBlockID) {
this.pipeline = pipeline;
this.containerBlockID = containerBlockID;
}
public Pipeline getPipeline() {
return pipeline;
}
public ContainerBlockID getBlockID() {
return containerBlockID;
}
}

36
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/BlockNotCommittedException.java
View File

@ -1,36 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdds.scm.container.common.helpers;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
/**
* Exceptions thrown when a block is yet to be committed on the datanode.
*/
public class BlockNotCommittedException extends StorageContainerException {
/**
* Constructs an {@code IOException} with the specified detail message.
*
* @param message The detail message (which is saved for later retrieval by
* the {@link #getMessage()} method)
*/
public BlockNotCommittedException(String message) {
super(message, ContainerProtos.Result.BLOCK_NOT_COMMITTED);
}
}

36
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/ContainerNotOpenException.java
View File

@ -1,36 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdds.scm.container.common.helpers;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
/**
* Exceptions thrown when a write/update opearation is done on non-open
* container.
*/
public class ContainerNotOpenException extends StorageContainerException {
/**
* Constructs an {@code IOException} with the specified detail message.
*
* @param message The detail message (which is saved for later retrieval by
* the {@link #getMessage()} method)
*/
public ContainerNotOpenException(String message) {
super(message, ContainerProtos.Result.CONTAINER_NOT_OPEN);
}
}

137
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/ContainerWithPipeline.java
View File

@ -1,137 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdds.scm.container.common.helpers;
import java.util.Comparator;
import org.apache.commons.lang3.builder.EqualsBuilder;
import org.apache.commons.lang3.builder.HashCodeBuilder;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.scm.container.ContainerInfo;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
import org.apache.hadoop.hdds.scm.pipeline.UnknownPipelineStateException;
/**
* Class wraps ozone container info.
*/
public class ContainerWithPipeline implements Comparator<ContainerWithPipeline>,
Comparable<ContainerWithPipeline> {
private final ContainerInfo containerInfo;
private final Pipeline pipeline;
public ContainerWithPipeline(ContainerInfo containerInfo, Pipeline pipeline) {
this.containerInfo = containerInfo;
this.pipeline = pipeline;
}
public ContainerInfo getContainerInfo() {
return containerInfo;
}
public Pipeline getPipeline() {
return pipeline;
}
public static ContainerWithPipeline fromProtobuf(
HddsProtos.ContainerWithPipeline allocatedContainer)
throws UnknownPipelineStateException {
return new ContainerWithPipeline(
ContainerInfo.fromProtobuf(allocatedContainer.getContainerInfo()),
Pipeline.getFromProtobuf(allocatedContainer.getPipeline()));
}
public HddsProtos.ContainerWithPipeline getProtobuf()
throws UnknownPipelineStateException {
HddsProtos.ContainerWithPipeline.Builder builder =
HddsProtos.ContainerWithPipeline.newBuilder();
builder.setContainerInfo(getContainerInfo().getProtobuf())
.setPipeline(getPipeline().getProtobufMessage());
return builder.build();
}
@Override
public String toString() {
return containerInfo.toString() + " | " + pipeline.toString();
}
@Override
public boolean equals(Object o) {
if (this == o) {
return true;
}
if (o == null || getClass() != o.getClass()) {
return false;
}
ContainerWithPipeline that = (ContainerWithPipeline) o;
return new EqualsBuilder()
.append(getContainerInfo(), that.getContainerInfo())
.append(getPipeline(), that.getPipeline())
.isEquals();
}
@Override
public int hashCode() {
return new HashCodeBuilder(11, 811)
.append(getContainerInfo())
.append(getPipeline())
.toHashCode();
}
/**
* Compares its two arguments for order. Returns a negative integer, zero, or
* a positive integer as the first argument is less than, equal to, or greater
* than the second.<p>
*
* @param o1 the first object to be compared.
* @param o2 the second object to be compared.
* @return a negative integer, zero, or a positive integer as the first
* argument is less than, equal to, or greater than the second.
* @throws NullPointerException if an argument is null and this comparator
* does not permit null arguments
* @throws ClassCastException if the arguments' types prevent them from
* being compared by this comparator.
*/
@Override
public int compare(ContainerWithPipeline o1, ContainerWithPipeline o2) {
return o1.getContainerInfo().compareTo(o2.getContainerInfo());
}
/**
* Compares this object with the specified object for order. Returns a
* negative integer, zero, or a positive integer as this object is less than,
* equal to, or greater than the specified object.
*
* @param o the object to be compared.
* @return a negative integer, zero, or a positive integer as this object is
* less than, equal to, or greater than the specified object.
* @throws NullPointerException if the specified object is null
* @throws ClassCastException if the specified object's type prevents it
* from being compared to this object.
*/
@Override
public int compareTo(ContainerWithPipeline o) {
return this.compare(this, o);
}
}

53
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/DeleteBlockResult.java
View File

@ -1,53 +0,0 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations under
* the License.
*/
package org.apache.hadoop.hdds.scm.container.common.helpers;
import org.apache.hadoop.hdds.client.BlockID;
import static org.apache.hadoop.hdds.protocol.proto
.ScmBlockLocationProtocolProtos.DeleteScmBlockResult;
/**
* Class wraps storage container manager block deletion results.
*/
public class DeleteBlockResult {
private BlockID blockID;
private DeleteScmBlockResult.Result result;
public DeleteBlockResult(final BlockID blockID,
final DeleteScmBlockResult.Result result) {
this.blockID = blockID;
this.result = result;
}
/**
* Get block id deleted.
* @return block id.
*/
public BlockID getBlockID() {
return blockID;
}
/**
* Get key deletion result.
* @return key deletion result.
*/
public DeleteScmBlockResult.Result getResult() {
return result;
}
}

111
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/ExcludeList.java
View File

@ -1,111 +0,0 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations under
* the License.
*/
package org.apache.hadoop.hdds.scm.container.common.helpers;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.scm.container.ContainerID;
import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
import java.util.ArrayList;
import java.util.List;
import java.util.Collection;
/**
* This class contains set of dns and containers which ozone client provides
* to be handed over to SCM when block allocation request comes.
*/
public class ExcludeList {
private final List<DatanodeDetails> datanodes;
private final List<ContainerID> containerIds;
private final List<PipelineID> pipelineIds;
public ExcludeList() {
datanodes = new ArrayList<>();
containerIds = new ArrayList<>();
pipelineIds = new ArrayList<>();
}
public List<ContainerID> getContainerIds() {
return containerIds;
}
public List<DatanodeDetails> getDatanodes() {
return datanodes;
}
public void addDatanodes(Collection<DatanodeDetails> dns) {
datanodes.addAll(dns);
}
public void addDatanode(DatanodeDetails dn) {
datanodes.add(dn);
}
public void addConatinerId(ContainerID containerId) {
containerIds.add(containerId);
}
public void addPipeline(PipelineID pipelineId) {
pipelineIds.add(pipelineId);
}
public List<PipelineID> getPipelineIds() {
return pipelineIds;
}
public HddsProtos.ExcludeListProto getProtoBuf() {
HddsProtos.ExcludeListProto.Builder builder =
HddsProtos.ExcludeListProto.newBuilder();
containerIds
.forEach(id -> builder.addContainerIds(id.getId()));
datanodes.forEach(dn -> {
builder.addDatanodes(dn.getUuidString());
});
pipelineIds.forEach(pipelineID -> {
builder.addPipelineIds(pipelineID.getProtobuf());
});
return builder.build();
}
public static ExcludeList getFromProtoBuf(
HddsProtos.ExcludeListProto excludeListProto) {
ExcludeList excludeList = new ExcludeList();
excludeListProto.getContainerIdsList().forEach(id -> {
excludeList.addConatinerId(ContainerID.valueof(id));
});
DatanodeDetails.Builder builder = DatanodeDetails.newBuilder();
excludeListProto.getDatanodesList().forEach(dn -> {
builder.setUuid(dn);
excludeList.addDatanode(builder.build());
});
excludeListProto.getPipelineIdsList().forEach(pipelineID -> {
excludeList.addPipeline(PipelineID.getFromProtobuf(pipelineID));
});
return excludeList;
}
public void clear() {
datanodes.clear();
containerIds.clear();
pipelineIds.clear();
}
}

35
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/InvalidContainerStateException.java
View File

@ -1,35 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdds.scm.container.common.helpers;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
/**
* Exceptions thrown when a container is in invalid state while doing a I/O.
*/
public class InvalidContainerStateException extends StorageContainerException {
/**
* Constructs an {@code IOException} with the specified detail message.
*
* @param message The detail message (which is saved for later retrieval by
* the {@link #getMessage()} method)
*/
public InvalidContainerStateException(String message) {
super(message, ContainerProtos.Result.INVALID_CONTAINER_STATE);
}
}

104
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/StorageContainerException.java
View File

@ -1,104 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdds.scm.container.common.helpers;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
import java.io.IOException;
/**
* Exceptions thrown from the Storage Container.
*/
public class StorageContainerException extends IOException {
private ContainerProtos.Result result;
/**
* Constructs an {@code IOException} with {@code null}
* as its error detail message.
*/
public StorageContainerException(ContainerProtos.Result result) {
this.result = result;
}
/**
* Constructs an {@code IOException} with the specified detail message.
*
* @param message The detail message (which is saved for later retrieval by
* the {@link #getMessage()} method)
* @param result - The result code
*/
public StorageContainerException(String message,
ContainerProtos.Result result) {
super(message);
this.result = result;
}
/**
* Constructs an {@code IOException} with the specified detail message
* and cause.
* <p>
* <p> Note that the detail message associated with {@code cause} is
* <i>not</i> automatically incorporated into this exception's detail
* message.
*
* @param message The detail message (which is saved for later retrieval by
* the {@link #getMessage()} method)
*
* @param cause The cause (which is saved for later retrieval by the {@link
* #getCause()} method). (A null value is permitted, and indicates that the
* cause is nonexistent or unknown.)
*
* @param result - The result code
* @since 1.6
*/
public StorageContainerException(String message, Throwable cause,
ContainerProtos.Result result) {
super(message, cause);
this.result = result;
}
/**
* Constructs an {@code IOException} with the specified cause and a
* detail message of {@code (cause==null ? null : cause.toString())}
* (which typically contains the class and detail message of {@code cause}).
* This constructor is useful for IO exceptions that are little more
* than wrappers for other throwables.
*
* @param cause The cause (which is saved for later retrieval by the {@link
* #getCause()} method). (A null value is permitted, and indicates that the
* cause is nonexistent or unknown.)
* @param result - The result code
* @since 1.6
*/
public StorageContainerException(Throwable cause, ContainerProtos.Result
result) {
super(cause);
this.result = result;
}
/**
* Returns Result.
*
* @return Result.
*/
public ContainerProtos.Result getResult() {
return result;
}
}

22
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/package-info.java
View File

@ -1,22 +0,0 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdds.scm.container.common.helpers;
/**
Contains protocol buffer helper classes and utilites used in
impl.
**/

18
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/package-info.java
View File

@ -1,18 +0,0 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdds.scm.container;

45
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/ContainerPlacementPolicy.java
View File

@ -1,45 +0,0 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations under
* the License.
*/
package org.apache.hadoop.hdds.scm.container.placement.algorithms;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import java.io.IOException;
import java.util.List;
/**
* A ContainerPlacementPolicy support choosing datanodes to build replication
* pipeline with specified constraints.
*/
public interface ContainerPlacementPolicy {
/**
* Given the replication factor and size required, return set of datanodes
* that satisfy the nodes and size requirement.
*
* @param excludedNodes - list of nodes to be excluded.
* @param favoredNodes - list of nodes preferred.
* @param nodesRequired - number of datanodes required.
* @param sizeRequired - size required for the container or block.
* @return list of datanodes chosen.
* @throws IOException
*/
List<DatanodeDetails> chooseDatanodes(List<DatanodeDetails> excludedNodes,
List<DatanodeDetails> favoredNodes, int nodesRequired, long sizeRequired)
throws IOException;
}

21
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/package-info.java
View File

@ -1,21 +0,0 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdds.scm.container.placement.algorithms;
/**
Contains container placement policy interface definition.
**/

127
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/exceptions/SCMException.java
View File

@ -1,127 +0,0 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdds.scm.exceptions;
import java.io.IOException;
/**
* Exception thrown by SCM.
*/
public class SCMException extends IOException {
private final ResultCodes result;
/**
* Constructs an {@code IOException} with {@code null}
* as its error detail message.
*/
public SCMException(ResultCodes result) {
this.result = result;
}
/**
* Constructs an {@code IOException} with the specified detail message.
*
* @param message The detail message (which is saved for later retrieval by
* the
* {@link #getMessage()} method)
*/
public SCMException(String message, ResultCodes result) {
super(message);
this.result = result;
}
/**
* Constructs an {@code IOException} with the specified detail message
* and cause.
* <p>
* <p> Note that the detail message associated with {@code cause} is
* <i>not</i> automatically incorporated into this exception's detail
* message.
*
* @param message The detail message (which is saved for later retrieval by
* the
* {@link #getMessage()} method)
* @param cause The cause (which is saved for later retrieval by the {@link
* #getCause()} method). (A null value is permitted, and indicates that the
* cause is nonexistent or unknown.)
* @since 1.6
*/
public SCMException(String message, Throwable cause, ResultCodes result) {
super(message, cause);
this.result = result;
}
/**
* Constructs an {@code IOException} with the specified cause and a
* detail message of {@code (cause==null ? null : cause.toString())}
* (which typically contains the class and detail message of {@code cause}).
* This constructor is useful for IO exceptions that are little more
* than wrappers for other throwables.
*
* @param cause The cause (which is saved for later retrieval by the {@link
* #getCause()} method). (A null value is permitted, and indicates that the
* cause is nonexistent or unknown.)
* @since 1.6
*/
public SCMException(Throwable cause, ResultCodes result) {
super(cause);
this.result = result;
}
/**
* Returns resultCode.
* @return ResultCode
*/
public ResultCodes getResult() {
return result;
}
/**
* Error codes to make it easy to decode these exceptions.
*/
public enum ResultCodes {
OK,
FAILED_TO_LOAD_NODEPOOL,
FAILED_TO_FIND_NODE_IN_POOL,
FAILED_TO_FIND_HEALTHY_NODES,
FAILED_TO_FIND_NODES_WITH_SPACE,
FAILED_TO_FIND_SUITABLE_NODE,
INVALID_CAPACITY,
INVALID_BLOCK_SIZE,
SAFE_MODE_EXCEPTION,
FAILED_TO_LOAD_OPEN_CONTAINER,
FAILED_TO_ALLOCATE_CONTAINER,
FAILED_TO_CHANGE_CONTAINER_STATE,
FAILED_TO_CHANGE_PIPELINE_STATE,
CONTAINER_EXISTS,
FAILED_TO_FIND_CONTAINER,
FAILED_TO_FIND_CONTAINER_WITH_SPACE,
BLOCK_EXISTS,
FAILED_TO_FIND_BLOCK,
IO_EXCEPTION,
UNEXPECTED_CONTAINER_STATE,
SCM_NOT_INITIALIZED,
DUPLICATE_DATANODE,
NO_SUCH_DATANODE,
NO_REPLICA_FOUND,
FAILED_TO_FIND_ACTIVE_PIPELINE,
FAILED_TO_INIT_CONTAINER_PLACEMENT_POLICY,
FAILED_TO_ALLOCATE_ENOUGH_BLOCKS,
INTERNAL_ERROR
}
}

21
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/exceptions/package-info.java
View File

@ -1,21 +0,0 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdds.scm.exceptions;
/**
Exception objects for the SCM Server.
*/

85
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/net/InnerNode.java
View File

@ -1,85 +0,0 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdds.scm.net;
import java.util.Collection;
import java.util.List;
/**
* The interface defines an inner node in a network topology.
* An inner node represents network topology entities, such as data center,
* rack, switch or logical group.
*/
public interface InnerNode extends Node {
/** A factory interface to get new InnerNode instance. */
interface Factory<N extends InnerNode> {
/** Construct an InnerNode from name, location, parent, level and cost. */
N newInnerNode(String name, String location, InnerNode parent, int level,
int cost);
}
/**
* Add node <i>n</i> to the subtree of this node.
* @param n node to be added
* @return true if the node is added; false otherwise
*/
boolean add(Node n);
/**
* Remove node <i>n</i> from the subtree of this node.
* @param n node to be deleted
*/
void remove(Node n);
/**
* Given a node's string representation, return a reference to the node.
* @param loc string location of the format /dc/rack/nodegroup/node
* @return null if the node is not found
*/
Node getNode(String loc);
/**
* @return number of its all nodes at level <i>level</i>. Here level is a
* relative level. If level is 1, means node itself. If level is 2, means its
* direct children, and so on.
**/
int getNumOfNodes(int level);
/**
* Get <i>leafIndex</i> leaf of this subtree.
*
* @param leafIndex an indexed leaf of the node
* @return the leaf node corresponding to the given index.
*/
Node getLeaf(int leafIndex);
/**
* Get <i>leafIndex</i> leaf of this subtree.
*
* @param leafIndex ode's index, start from 0, skip the nodes in
* excludedScope and excludedNodes with ancestorGen
* @param excludedScopes the excluded scopes
* @param excludedNodes nodes to be excluded. If ancestorGen is not 0,
* the chosen node will not share same ancestor with
* those in excluded nodes at the specified generation
* @param ancestorGen ignored with value is 0
* @return the leaf node corresponding to the given index
*/
Node getLeaf(int leafIndex, List<String> excludedScopes,
Collection<Node> excludedNodes, int ancestorGen);
}

509
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/net/InnerNodeImpl.java
View File

@ -1,509 +0,0 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdds.scm.net;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import com.google.common.base.Preconditions;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import static org.apache.hadoop.hdds.scm.net.NetConstants.PATH_SEPARATOR_STR;
import static org.apache.hadoop.hdds.scm.net.NetConstants.PATH_SEPARATOR;
/**
* A thread safe class that implements InnerNode interface.
*/
public class InnerNodeImpl extends NodeImpl implements InnerNode {
protected static class Factory implements InnerNode.Factory<InnerNodeImpl> {
protected Factory() {}
public InnerNodeImpl newInnerNode(String name, String location,
InnerNode parent, int level, int cost) {
return new InnerNodeImpl(name, location, parent, level, cost);
}
}
static final Factory FACTORY = new Factory();
// a map of node's network name to Node for quick search and keep
// the insert order
private final HashMap<String, Node> childrenMap =
new LinkedHashMap<String, Node>();
// number of descendant leaves under this node
private int numOfLeaves;
// LOGGER
public static final Logger LOG = LoggerFactory.getLogger(InnerNodeImpl.class);
/**
* Construct an InnerNode from its name, network location, parent, level and
* its cost.
**/
protected InnerNodeImpl(String name, String location, InnerNode parent,
int level, int cost) {
super(name, location, parent, level, cost);
}
/** @return the number of children this node has */
private int getNumOfChildren() {
return childrenMap.size();
}
/** @return its leaf nodes number */
@Override
public int getNumOfLeaves() {
return numOfLeaves;
}
/**
* @return number of its all nodes at level <i>level</i>. Here level is a
* relative level. If level is 1, means node itself. If level is 2, means its
* direct children, and so on.
**/
public int getNumOfNodes(int level) {
Preconditions.checkArgument(level > 0);
int count = 0;
if (level == 1) {
count += 1;
} else if (level == 2) {
count += getNumOfChildren();
} else {
for (Node node: childrenMap.values()) {
if (node instanceof InnerNode) {
count += ((InnerNode)node).getNumOfNodes(level -1);
} else {
throw new RuntimeException("Cannot support Level:" + level +
" on this node " + this.toString());
}
}
}
return count;
}
/**
* Judge if this node is the parent of a leave node <i>n</i>.
* @return true if this node is the parent of <i>n</i>
*/
private boolean isLeafParent() {
if (childrenMap.isEmpty()) {
return true;
}
Node child = childrenMap.values().iterator().next();
return child instanceof InnerNode ? false : true;
}
/**
* Judge if this node is the parent of node <i>node</i>.
* @param node a node
* @return true if this node is the parent of <i>n</i>
*/
private boolean isParent(Node node) {
return node.getNetworkLocation().equals(this.getNetworkFullPath());
}
/**
* Add node <i>node</i> to the subtree of this node.
* @param node node to be added
* @return true if the node is added, false if is only updated
*/
public boolean add(Node node) {
if (!isAncestor(node)) {
throw new IllegalArgumentException(node.getNetworkName()
+ ", which is located at " + node.getNetworkLocation()
+ ", is not a descendant of " + this.getNetworkFullPath());
}
if (isParent(node)) {
// this node is the parent, then add it directly
node.setParent(this);
node.setLevel(this.getLevel() + 1);
Node current = childrenMap.put(node.getNetworkName(), node);
if (current != null) {
return false;
}
} else {
// find the next level ancestor node
String ancestorName = getNextLevelAncestorName(node);
InnerNode childNode = (InnerNode)childrenMap.get(ancestorName);
if (childNode == null) {
// create a new InnerNode for this ancestor node
childNode = createChildNode(ancestorName);
childrenMap.put(childNode.getNetworkName(), childNode);
}
// add node to the subtree of the next ancestor node
if (!childNode.add(node)) {
return false;
}
}
numOfLeaves++;
return true;
}
/**
* Remove node <i>node</i> from the subtree of this node.
* @param node node to be deleted
*/
public void remove(Node node) {
if (!isAncestor(node)) {
throw new IllegalArgumentException(node.getNetworkName()
+ ", which is located at " + node.getNetworkLocation()
+ ", is not a descendant of " + this.getNetworkFullPath());
}
if (isParent(node)) {
// this node is the parent, remove it directly
if (childrenMap.containsKey(node.getNetworkName())) {
childrenMap.remove(node.getNetworkName());
node.setParent(null);
} else {
throw new RuntimeException("Should not come to here. Node:" +
node.getNetworkFullPath() + ", Parent:" +
this.getNetworkFullPath());
}
} else {
// find the next ancestor node
String ancestorName = getNextLevelAncestorName(node);
InnerNodeImpl childNode = (InnerNodeImpl)childrenMap.get(ancestorName);
Preconditions.checkNotNull(childNode, "InnerNode is deleted before leaf");
// remove node from the parent node
childNode.remove(node);
// if the parent node has no children, remove the parent node too
if (childNode.getNumOfChildren() == 0) {
childrenMap.remove(ancestorName);
}
}
numOfLeaves--;
}
/**
* Given a node's string representation, return a reference to the node.
* Node can be leaf node or inner node.
*
* @param loc string location of a node. If loc starts with "/", it's a
* absolute path, otherwise a relative path. Following examples
* are all accepted,
* 1. /dc1/rm1/rack1 -> an inner node
* 2. /dc1/rm1/rack1/node1 -> a leaf node
* 3. rack1/node1 -> a relative path to this node
*
* @return null if the node is not found
*/
public Node getNode(String loc) {
if (loc == null) {
return null;
}
String fullPath = this.getNetworkFullPath();
if (loc.equalsIgnoreCase(fullPath)) {
return this;
}
// remove current node's location from loc when it's a absolute path
if (fullPath.equals(NetConstants.PATH_SEPARATOR_STR)) {
// current node is ROOT
if (loc.startsWith(PATH_SEPARATOR_STR)) {
loc = loc.substring(1);
}
} else if (loc.startsWith(fullPath)) {
loc = loc.substring(fullPath.length());
// skip the separator "/"
loc = loc.substring(1);
}
String[] path = loc.split(PATH_SEPARATOR_STR, 2);
Node child = childrenMap.get(path[0]);
if (child == null) {
return null;
}
if (path.length == 1){
return child;
}
if (child instanceof InnerNode) {
return ((InnerNode)child).getNode(path[1]);
} else {
return null;
}
}
/**
* get <i>leafIndex</i> leaf of this subtree.
*
* @param leafIndex an indexed leaf of the node
* @return the leaf node corresponding to the given index.
*/
public Node getLeaf(int leafIndex) {
Preconditions.checkArgument(leafIndex >= 0);
// children are leaves
if (isLeafParent()) {
// range check
if (leafIndex >= getNumOfChildren()) {
return null;
}
return getChildNode(leafIndex);
} else {
for(Node node : childrenMap.values()) {
InnerNodeImpl child = (InnerNodeImpl)node;
int leafCount = child.getNumOfLeaves();
if (leafIndex < leafCount) {
return child.getLeaf(leafIndex);
} else {
leafIndex -= leafCount;
}
}
return null;
}
}
/**
* Get <i>leafIndex</i> leaf of this subtree.
*
* @param leafIndex node's index, start from 0, skip the nodes in
* excludedScope and excludedNodes with ancestorGen
* @param excludedScopes the exclude scopes
* @param excludedNodes nodes to be excluded from. If ancestorGen is not 0,
* the chosen node will not share same ancestor with
* those in excluded nodes at the specified generation
* @param ancestorGen apply to excludeNodes, when value is 0, then no same
* ancestor enforcement on excludedNodes
* @return the leaf node corresponding to the given index.
* Example:
*
* / --- root
* / \
* / \
* / \
* / \
* dc1 dc2
* / \ / \
* / \ / \
* / \ / \
* rack1 rack2 rack1 rack2
* / \ / \ / \ / \
* n1 n2 n3 n4 n5 n6 n7 n8
*
* Input:
* leafIndex = 2
* excludedScope = /dc2/rack2
* excludedNodes = {/dc1/rack1/n1}
* ancestorGen = 1
*
* Output:
* node /dc1/rack2/n5
*
* Explanation:
* Since excludedNodes is n1 and ancestorGen is 1, it means nodes under
* /root/dc1/rack1 are excluded. Given leafIndex start from 0, LeafIndex 2
* means picking the 3th available node, which is n5.
*
*/
public Node getLeaf(int leafIndex, List<String> excludedScopes,
Collection<Node> excludedNodes, int ancestorGen) {
Preconditions.checkArgument(leafIndex >= 0 && ancestorGen >= 0);
// come to leaf parent layer
if (isLeafParent()) {
return getLeafOnLeafParent(leafIndex, excludedScopes, excludedNodes);
}
int maxLevel = NodeSchemaManager.getInstance().getMaxLevel();
// this node's children, it's generation as the ancestor of the leaf node
int currentGen = maxLevel - this.getLevel() - 1;
// build an ancestor(children) to exclude node count map
Map<Node, Integer> countMap =
getAncestorCountMap(excludedNodes, ancestorGen, currentGen);
// nodes covered by excluded scope
Map<String, Integer> excludedNodeCount =
getExcludedScopeNodeCount(excludedScopes);
for (Node child : childrenMap.values()) {
int leafCount = child.getNumOfLeaves();
// skip nodes covered by excluded scopes
for (Map.Entry<String, Integer> entry: excludedNodeCount.entrySet()) {
if (entry.getKey().startsWith(child.getNetworkFullPath())) {
leafCount -= entry.getValue();
}
}
// skip nodes covered by excluded nodes and ancestorGen
Integer count = countMap.get(child);
if (count != null) {
leafCount -= count;
}
if (leafIndex < leafCount) {
return ((InnerNode)child).getLeaf(leafIndex, excludedScopes,
excludedNodes, ancestorGen);
} else {
leafIndex -= leafCount;
}
}
return null;
}
@Override
public boolean equals(Object to) {
if (to == null) {
return false;
}
if (this == to) {
return true;
}
return this.toString().equals(to.toString());
}
@Override
public int hashCode() {
return super.hashCode();
}
/**
* Get a ancestor to its excluded node count map.
*
* @param nodes a collection of leaf nodes to exclude
* @param genToExclude the ancestor generation to exclude
* @param genToReturn the ancestor generation to return the count map
* @return the map.
* example:
*
* * --- root
* / \
* * * -- genToReturn =2
* / \ / \
* * * * * -- genToExclude = 1
* /\ /\ /\ /\
* * * * * * * * * -- nodes
*/
private Map<Node, Integer> getAncestorCountMap(Collection<Node> nodes,
int genToExclude, int genToReturn) {
Preconditions.checkState(genToExclude >= 0);
Preconditions.checkState(genToReturn >= 0);
if (nodes == null || nodes.size() == 0) {
return Collections.emptyMap();
}
// with the recursive call, genToReturn can be smaller than genToExclude
if (genToReturn < genToExclude) {
genToExclude = genToReturn;
}
// ancestorToExclude to ancestorToReturn map
HashMap<Node, Node> ancestorMap = new HashMap<>();
for (Node node: nodes) {
Node ancestorToExclude = node.getAncestor(genToExclude);
Node ancestorToReturn = node.getAncestor(genToReturn);
if (ancestorToExclude == null || ancestorToReturn == null) {
LOG.warn("Ancestor not found, node: " + node.getNetworkFullPath() +
", generation to exclude: " + genToExclude +
", generation to return:" + genToReturn);
continue;
}
ancestorMap.put(ancestorToExclude, ancestorToReturn);
}
// ancestorToReturn to exclude node count map
HashMap<Node, Integer> countMap = new HashMap<>();
for (Map.Entry<Node, Node> entry : ancestorMap.entrySet()) {
countMap.compute(entry.getValue(),
(key, n) -> (n == null ? 0 : n) + entry.getKey().getNumOfLeaves());
}
return countMap;
}
/**
* Get the node with leafIndex, considering skip nodes in excludedScope
* and in excludeNodes list.
*/
private Node getLeafOnLeafParent(int leafIndex, List<String> excludedScopes,
Collection<Node> excludedNodes) {
Preconditions.checkArgument(isLeafParent() && leafIndex >= 0);
if (leafIndex >= getNumOfChildren()) {
return null;
}
for(Node node : childrenMap.values()) {
if (excludedNodes != null && excludedNodes.contains(node)) {
continue;
}
if (excludedScopes != null && excludedScopes.size() > 0) {
if (excludedScopes.stream().anyMatch(scope ->
node.getNetworkFullPath().startsWith(scope))) {
continue;
}
}
if (leafIndex == 0) {
return node;
}
leafIndex--;
}
return null;
}
/**
* Return child's name of this node which is an ancestor of node <i>n</i>.
*/
private String getNextLevelAncestorName(Node n) {
int parentPathLen = this.getNetworkFullPath().length();
String name = n.getNetworkLocation().substring(parentPathLen);
if (name.charAt(0) == PATH_SEPARATOR) {
name = name.substring(1);
}
int index = name.indexOf(PATH_SEPARATOR);
if (index != -1) {
name = name.substring(0, index);
}
return name;
}
/**
* Creates a child node to be added to the list of children.
* @param name The name of the child node
* @return A new inner node
* @see InnerNodeImpl(String, String, InnerNode, int)
*/
private InnerNodeImpl createChildNode(String name) {
int childLevel = this.getLevel() + 1;
int cost = NodeSchemaManager.getInstance().getCost(childLevel);
return new InnerNodeImpl(name, this.getNetworkFullPath(), this, childLevel,
cost);
}
/** Get node with index <i>index</i>. */
private Node getChildNode(int index) {
Iterator iterator = childrenMap.values().iterator();
Node node = null;
while(index >= 0 && iterator.hasNext()) {
node = (Node)iterator.next();
index--;
}
return node;
}
/** Get how many leaf nodes are covered by the excludedScopes(no overlap). */
private Map<String, Integer> getExcludedScopeNodeCount(
List<String> excludedScopes) {
HashMap<String, Integer> nodeCounts = new HashMap<>();
if (excludedScopes == null || excludedScopes.isEmpty()) {
return nodeCounts;
}
for (String scope: excludedScopes) {
Node excludedScopeNode = getNode(scope);
nodeCounts.put(scope, excludedScopeNode == null ? 0 :
excludedScopeNode.getNumOfLeaves());
}
return nodeCounts;
}
}

67
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/net/NetConstants.java
View File

@ -1,67 +0,0 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdds.scm.net;
import org.apache.hadoop.hdds.scm.net.NodeSchema.LayerType;
/**
* Class to hold network topology related constants and configurations.
*/
public final class NetConstants {
private NetConstants() {
// Prevent instantiation
}
public final static char PATH_SEPARATOR = '/';
/** Path separator as a string. */
public final static String PATH_SEPARATOR_STR = "/";
public final static String SCOPE_REVERSE_STR = "~";
/** string representation of root. */
public final static String ROOT = "";
public final static int INNER_NODE_COST_DEFAULT = 1;
public final static int NODE_COST_DEFAULT = 0;
public final static int ANCESTOR_GENERATION_DEFAULT = 0;
public final static int ROOT_LEVEL = 1;
public final static String NODE_COST_PREFIX = "$";
public final static String DEFAULT_RACK = "/default-rack";
public final static String DEFAULT_NODEGROUP = "/default-nodegroup";
public final static String DEFAULT_DATACENTER = "/default-datacenter";
public final static String DEFAULT_REGION = "/default-dataregion";
// Build-in network topology node schema
public static final NodeSchema ROOT_SCHEMA =
new NodeSchema.Builder().setType(LayerType.ROOT).build();
public static final NodeSchema REGION_SCHEMA =
new NodeSchema.Builder().setType(LayerType.INNER_NODE)
.setDefaultName(DEFAULT_REGION).build();
public static final NodeSchema DATACENTER_SCHEMA =
new NodeSchema.Builder().setType(LayerType.INNER_NODE)
.setDefaultName(DEFAULT_DATACENTER).build();
public static final NodeSchema RACK_SCHEMA =
new NodeSchema.Builder().setType(LayerType.INNER_NODE)
.setDefaultName(DEFAULT_RACK).build();
public static final NodeSchema NODEGROUP_SCHEMA =
new NodeSchema.Builder().setType(LayerType.INNER_NODE)
.setDefaultName(DEFAULT_NODEGROUP).build();
public static final NodeSchema LEAF_SCHEMA =
new NodeSchema.Builder().setType(LayerType.LEAF_NODE).build();
}

161
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/net/NetUtils.java
View File

@ -1,161 +0,0 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdds.scm.net;
import org.apache.commons.collections.CollectionUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Iterator;
import java.util.List;
import java.util.stream.Collectors;
/**
* Utility class to facilitate network topology functions.
*/
public final class NetUtils {
public static final Logger LOG = LoggerFactory.getLogger(NetUtils.class);
private NetUtils() {
// Prevent instantiation
}
/**
* Normalize a path by stripping off any trailing.
* {@link NetConstants#PATH_SEPARATOR}
* @param path path to normalize.
* @return the normalised path
* If <i>path</i>is empty or null, then {@link NetConstants#ROOT} is returned
*/
public static String normalize(String path) {
if (path == null || path.length() == 0) {
return NetConstants.ROOT;
}
if (path.charAt(0) != NetConstants.PATH_SEPARATOR) {
throw new IllegalArgumentException(
"Network Location path does not start with "
+ NetConstants.PATH_SEPARATOR_STR + ": " + path);
}
// Remove any trailing NetConstants.PATH_SEPARATOR
return path.length() == 1 ? path :
path.replaceAll(NetConstants.PATH_SEPARATOR_STR + "+$", "");
}
/**
* Given a network topology location string, return its network topology
* depth, E.g. the depth of /dc1/rack1/ng1/node1 is 5.
*/
public static int locationToDepth(String location) {
String newLocation = normalize(location);
return newLocation.equals(NetConstants.PATH_SEPARATOR_STR) ? 1 :
newLocation.split(NetConstants.PATH_SEPARATOR_STR).length;
}
/**
* Remove node from mutableExcludedNodes if it's covered by excludedScope.
* Please noted that mutableExcludedNodes content might be changed after the
* function call.
*/
public static void removeDuplicate(NetworkTopology topology,
Collection<Node> mutableExcludedNodes, List<String> mutableExcludedScopes,
int ancestorGen) {
if (CollectionUtils.isEmpty(mutableExcludedNodes) ||
CollectionUtils.isEmpty(mutableExcludedScopes) || topology == null) {
return;
}
Iterator<Node> iterator = mutableExcludedNodes.iterator();
while (iterator.hasNext() && (!mutableExcludedScopes.isEmpty())) {
Node node = iterator.next();
Node ancestor = topology.getAncestor(node, ancestorGen);
if (ancestor == null) {
LOG.warn("Fail to get ancestor generation " + ancestorGen +
" of node :" + node);
continue;
}
// excludedScope is child of ancestor
List<String> duplicateList = mutableExcludedScopes.stream()
.filter(scope -> scope.startsWith(ancestor.getNetworkFullPath()))
.collect(Collectors.toList());
mutableExcludedScopes.removeAll(duplicateList);
// ancestor is covered by excludedScope
mutableExcludedScopes.stream().forEach(scope -> {
if (ancestor.getNetworkFullPath().startsWith(scope)) {
// remove exclude node if it's covered by excludedScope
iterator.remove();
}
});
}
}
/**
* Remove node from mutableExcludedNodes if it's not part of scope
* Please noted that mutableExcludedNodes content might be changed after the
* function call.
*/
public static void removeOutscope(Collection<Node> mutableExcludedNodes,
String scope) {
if (CollectionUtils.isEmpty(mutableExcludedNodes) || scope == null) {
return;
}
synchronized (mutableExcludedNodes) {
Iterator<Node> iterator = mutableExcludedNodes.iterator();
while (iterator.hasNext()) {
Node next = iterator.next();
if (!next.getNetworkFullPath().startsWith(scope)) {
iterator.remove();
}
}
}
}
/**
* Get a ancestor list for nodes on generation <i>generation</i>.
*
* @param nodes a collection of leaf nodes
* @param generation the ancestor generation
* @return the ancestor list. If no ancestor is found, then a empty list is
* returned.
*/
public static List<Node> getAncestorList(NetworkTopology topology,
Collection<Node> nodes, int generation) {
List<Node> ancestorList = new ArrayList<>();
if (topology == null || CollectionUtils.isEmpty(nodes) ||
generation == 0) {
return ancestorList;
}
Iterator<Node> iterator = nodes.iterator();
while (iterator.hasNext()) {
Node node = iterator.next();
Node ancestor = topology.getAncestor(node, generation);
if (ancestor == null) {
LOG.warn("Fail to get ancestor generation " + generation +
" of node :" + node);
continue;
}
if (!ancestorList.contains(ancestor)) {
ancestorList.add(ancestor);
}
}
return ancestorList;
}
}

229
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/net/NetworkTopology.java
View File

@ -1,229 +0,0 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdds.scm.net;
import java.util.Collection;
import java.util.List;
/**
* The interface defines a network topology.
*/
public interface NetworkTopology {
/** Exception for invalid network topology detection. */
class InvalidTopologyException extends RuntimeException {
private static final long serialVersionUID = 1L;
public InvalidTopologyException(String msg) {
super(msg);
}
}
/**
* Add a leaf node. This will be called when a new datanode is added.
* @param node node to be added; can be null
* @exception IllegalArgumentException if add a node to a leave or node to be
* added is not a leaf
*/
void add(Node node);
/**
* Remove a node from the network topology. This will be called when a
* existing datanode is removed from the system.
* @param node node to be removed; cannot be null
*/
void remove(Node node);
/**
* Check if the tree already contains node <i>node</i>.
* @param node a node
* @return true if <i>node</i> is already in the tree; false otherwise
*/
boolean contains(Node node);
/**
* Compare the direct parent of each node for equality.
* @return true if their parent are the same
*/
boolean isSameParent(Node node1, Node node2);
/**
* Compare the specified ancestor generation of each node for equality.
* ancestorGen 1 means parent.
* @return true if their specified generation ancestor are equal
*/
boolean isSameAncestor(Node node1, Node node2, int ancestorGen);
/**
* Get the ancestor for node on generation <i>ancestorGen</i>.
*
* @param node the node to get ancestor
* @param ancestorGen the ancestor generation
* @return the ancestor. If no ancestor is found, then null is returned.
*/
Node getAncestor(Node node, int ancestorGen);
/**
* Return the max level of this topology, start from 1 for ROOT. For example,
* topology like "/rack/node" has the max level '3'.
*/
int getMaxLevel();
/**
* Given a string representation of a node, return its reference.
* @param loc a path string representing a node, can be leaf or inner node
* @return a reference to the node; null if the node is not in the tree
*/
Node getNode(String loc);
/**
* Given a string representation of a InnerNode, return its leaf nodes count.
* @param loc a path-like string representation of a InnerNode
* @return the number of leaf nodes, 0 if it's not an InnerNode or the node
* doesn't exist
*/
int getNumOfLeafNode(String loc);
/**
* Return the node numbers at level <i>level</i>.
* @param level topology level, start from 1, which means ROOT
* @return the number of nodes on the level
*/
int getNumOfNodes(int level);
/**
* Randomly choose a node in the scope.
* @param scope range of nodes from which a node will be chosen. If scope
* starts with ~, choose one from the all nodes except for the
* ones in <i>scope</i>; otherwise, choose one from <i>scope</i>.
* @return the chosen node
*/
Node chooseRandom(String scope);
/**
* Randomly choose a node in the scope, ano not in the exclude scope.
* @param scope range of nodes from which a node will be chosen. cannot start
* with ~
* @param excludedScopes the chosen nodes cannot be in these ranges. cannot
* starts with ~
* @return the chosen node
*/
Node chooseRandom(String scope, List<String> excludedScopes);
/**
* Randomly choose a leaf node from <i>scope</i>.
*
* If scope starts with ~, choose one from the all nodes except for the
* ones in <i>scope</i>; otherwise, choose nodes from <i>scope</i>.
* If excludedNodes is given, choose a node that's not in excludedNodes.
*
* @param scope range of nodes from which a node will be chosen
* @param excludedNodes nodes to be excluded
*
* @return the chosen node
*/
Node chooseRandom(String scope, Collection<Node> excludedNodes);
/**
* Randomly choose a leaf node from <i>scope</i>.
*
* If scope starts with ~, choose one from the all nodes except for the
* ones in <i>scope</i>; otherwise, choose nodes from <i>scope</i>.
* If excludedNodes is given, choose a node that's not in excludedNodes.
*
* @param scope range of nodes from which a node will be chosen
* @param excludedNodes nodes to be excluded from.
* @param ancestorGen matters when excludeNodes is not null. It means the
* ancestor generation that's not allowed to share between chosen node and the
* excludedNodes. For example, if ancestorGen is 1, means chosen node
* cannot share the same parent with excludeNodes. If value is 2, cannot
* share the same grand parent, and so on. If ancestorGen is 0, then no
* effect.
*
* @return the chosen node
*/
Node chooseRandom(String scope, Collection<Node> excludedNodes,
int ancestorGen);
/**
* Randomly choose one node from <i>scope</i>, share the same generation
* ancestor with <i>affinityNode</i>, and exclude nodes in
* <i>excludeScope</i> and <i>excludeNodes</i>.
*
* @param scope range of nodes from which a node will be chosen, cannot start
* with ~
* @param excludedScopes ranges of nodes to be excluded, cannot start with ~
* @param excludedNodes nodes to be excluded
* @param affinityNode when not null, the chosen node should share the same
* ancestor with this node at generation ancestorGen.
* Ignored when value is null
* @param ancestorGen If 0, then no same generation ancestor enforcement on
* both excludedNodes and affinityNode. If greater than 0,
* then apply to affinityNode(if not null), or apply to
* excludedNodes if affinityNode is null
* @return the chosen node
*/
Node chooseRandom(String scope, List<String> excludedScopes,
Collection<Node> excludedNodes, Node affinityNode, int ancestorGen);
/**
* Choose the node at index <i>index</i> from <i>scope</i>, share the same
* generation ancestor with <i>affinityNode</i>, and exclude nodes in
* <i>excludeScope</i> and <i>excludeNodes</i>.
*
* @param leafIndex node index, exclude nodes in excludedScope and
* excludedNodes
* @param scope range of nodes from which a node will be chosen, cannot start
* with ~
* @param excludedScopes ranges of nodes to be excluded, cannot start with ~
* @param excludedNodes nodes to be excluded
* @param affinityNode when not null, the chosen node should share the same
* ancestor with this node at generation ancestorGen.
* Ignored when value is null
* @param ancestorGen If 0, then no same generation ancestor enforcement on
* both excludedNodes and affinityNode. If greater than 0,
* then apply to affinityNode(if not null), or apply to
* excludedNodes if affinityNode is null
* @return the chosen node
*/
Node getNode(int leafIndex, String scope, List<String> excludedScopes,
Collection<Node> excludedNodes, Node affinityNode, int ancestorGen);
/** Return the distance cost between two nodes
* The distance cost from one node to its parent is it's parent's cost
* The distance cost between two nodes is calculated by summing up their
* distances cost to their closest common ancestor.
* @param node1 one node
* @param node2 another node
* @return the distance cost between node1 and node2 which is zero if they
* are the same or {@link Integer#MAX_VALUE} if node1 or node2 do not belong
* to the cluster
*/
int getDistanceCost(Node node1, Node node2);
/**
* Sort nodes array by network distance to <i>reader</i> to reduces network
* traffic and improves performance.
*
* As an additional twist, we also randomize the nodes at each network
* distance. This helps with load balancing when there is data skew.
*
* @param reader Node where need the data
* @param nodes Available replicas with the requested data
* @param activeLen Number of active nodes at the front of the array
*/
List<? extends Node> sortByDistanceCost(Node reader,
List<? extends Node> nodes, int activeLen);
}

798
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/net/NetworkTopologyImpl.java
View File

@ -1,798 +0,0 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdds.scm.net;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
import org.apache.commons.collections.CollectionUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.TreeMap;
import java.util.concurrent.ThreadLocalRandom;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.stream.Collectors;
import org.apache.hadoop.conf.Configuration;
import static org.apache.hadoop.hdds.scm.net.NetConstants.ROOT;
import static org.apache.hadoop.hdds.scm.net.NetConstants.SCOPE_REVERSE_STR;
import static org.apache.hadoop.hdds.scm.net.NetConstants.ANCESTOR_GENERATION_DEFAULT;
/**
* The class represents a cluster of computers with a tree hierarchical
* network topology. In the network topology, leaves represent data nodes
* (computers) and inner nodes represent datacenter/core-switches/routers that
* manages traffic in/out of data centers or racks.
*/
public class NetworkTopologyImpl implements NetworkTopology{
public static final Logger LOG =
LoggerFactory.getLogger(NetworkTopology.class);
/** The Inner node crate factory. */
private final InnerNode.Factory factory;
/** The root cluster tree. */
private final InnerNode clusterTree;
/** Depth of all leaf nodes. */
private final int maxLevel;
/** Schema manager. */
private final NodeSchemaManager schemaManager;
/** Lock to coordinate cluster tree access. */
private ReadWriteLock netlock = new ReentrantReadWriteLock(true);
public NetworkTopologyImpl(Configuration conf) {
schemaManager = NodeSchemaManager.getInstance();
schemaManager.init(conf);
maxLevel = schemaManager.getMaxLevel();
factory = InnerNodeImpl.FACTORY;
clusterTree = factory.newInnerNode(ROOT, null, null,
NetConstants.ROOT_LEVEL,
schemaManager.getCost(NetConstants.ROOT_LEVEL));
}
@VisibleForTesting
public NetworkTopologyImpl(NodeSchemaManager manager) {
schemaManager = manager;
maxLevel = schemaManager.getMaxLevel();
factory = InnerNodeImpl.FACTORY;
clusterTree = factory.newInnerNode(ROOT, null, null,
NetConstants.ROOT_LEVEL,
schemaManager.getCost(NetConstants.ROOT_LEVEL));
}
/**
* Add a leaf node. This will be called when a new datanode is added.
* @param node node to be added; can be null
* @exception IllegalArgumentException if add a node to a leave or node to be
* added is not a leaf
*/
public void add(Node node) {
Preconditions.checkArgument(node != null, "node cannot be null");
if (node instanceof InnerNode) {
throw new IllegalArgumentException(
"Not allowed to add an inner node: "+ node.getNetworkFullPath());
}
int newDepth = NetUtils.locationToDepth(node.getNetworkLocation()) + 1;
// Check depth
if (maxLevel != newDepth) {
throw new InvalidTopologyException("Failed to add " +
node.getNetworkFullPath() + ": Its path depth is not " + maxLevel);
}
netlock.writeLock().lock();
boolean add;
try {
add = clusterTree.add(node);
}finally {
netlock.writeLock().unlock();
}
if (add) {
LOG.info("Added a new node: " + node.getNetworkFullPath());
if (LOG.isDebugEnabled()) {
LOG.debug("NetworkTopology became:\n{}", this);
}
}
}
/**
* Remove a node from the network topology. This will be called when a
* existing datanode is removed from the system.
* @param node node to be removed; cannot be null
*/
public void remove(Node node) {
Preconditions.checkArgument(node != null, "node cannot be null");
if (node instanceof InnerNode) {
throw new IllegalArgumentException(
"Not allowed to remove an inner node: "+ node.getNetworkFullPath());
}
netlock.writeLock().lock();
try {
clusterTree.remove(node);
}finally {
netlock.writeLock().unlock();
}
LOG.info("Removed a node: " + node.getNetworkFullPath());
if (LOG.isDebugEnabled()) {
LOG.debug("NetworkTopology became:\n{}", this);
}
}
/**
* Check if the tree already contains node <i>node</i>.
* @param node a node
* @return true if <i>node</i> is already in the tree; false otherwise
*/
public boolean contains(Node node) {
Preconditions.checkArgument(node != null, "node cannot be null");
netlock.readLock().lock();
try {
Node parent = node.getParent();
while (parent != null && parent != clusterTree) {
parent = parent.getParent();
}
if (parent == clusterTree) {
return true;
}
} finally {
netlock.readLock().unlock();
}
return false;
}
/**
* Compare the specified ancestor generation of each node for equality.
* @return true if their specified generation ancestor are equal
*/
public boolean isSameAncestor(Node node1, Node node2, int ancestorGen) {
if (node1 == null || node2 == null || ancestorGen <= 0) {
return false;
}
netlock.readLock().lock();
try {
return node1.getAncestor(ancestorGen) == node2.getAncestor(ancestorGen);
} finally {
netlock.readLock().unlock();
}
}
/**
* Compare the direct parent of each node for equality.
* @return true if their parent are the same
*/
public boolean isSameParent(Node node1, Node node2) {
if (node1 == null || node2 == null) {
return false;
}
netlock.readLock().lock();
try {
node1 = node1.getParent();
node2 = node2.getParent();
return node1 == node2;
} finally {
netlock.readLock().unlock();
}
}
/**
* Get the ancestor for node on generation <i>ancestorGen</i>.
*
* @param node the node to get ancestor
* @param ancestorGen the ancestor generation
* @return the ancestor. If no ancestor is found, then null is returned.
*/
public Node getAncestor(Node node, int ancestorGen) {
if (node == null) {
return null;
}
netlock.readLock().lock();
try {
return node.getAncestor(ancestorGen);
} finally {
netlock.readLock().unlock();
}
}
/**
* Given a string representation of a node(leaf or inner), return its
* reference.
* @param loc a path string representing a node, can be leaf or inner node
* @return a reference to the node, null if the node is not in the tree
*/
public Node getNode(String loc) {
loc = NetUtils.normalize(loc);
netlock.readLock().lock();
try {
if (!ROOT.equals(loc)) {
return clusterTree.getNode(loc);
} else {
return clusterTree;
}
} finally {
netlock.readLock().unlock();
}
}
/**
* Given a string representation of Node, return its leaf nodes count.
* @param loc a path-like string representation of Node
* @return the number of leaf nodes for InnerNode, 1 for leaf node, 0 if node
* doesn't exist
*/
public int getNumOfLeafNode(String loc) {
netlock.readLock().lock();
try {
Node node = getNode(loc);
if (node != null) {
return node.getNumOfLeaves();
}
} finally {
netlock.readLock().unlock();
}
return 0;
}
/**
* Return the max level of this tree, start from 1 for ROOT. For example,
* topology like "/rack/node" has the max level '3'.
*/
public int getMaxLevel() {
return maxLevel;
}
/**
* Return the node numbers at level <i>level</i>.
* @param level topology level, start from 1, which means ROOT
* @return the number of nodes on the level
*/
public int getNumOfNodes(int level) {
Preconditions.checkArgument(level > 0 && level <= maxLevel,
"Invalid level");
netlock.readLock().lock();
try {
return clusterTree.getNumOfNodes(level);
} finally {
netlock.readLock().unlock();
}
}
/**
* Randomly choose a node in the scope.
* @param scope range of nodes from which a node will be chosen. If scope
* starts with ~, choose one from the all nodes except for the
* ones in <i>scope</i>; otherwise, choose one from <i>scope</i>.
* @return the chosen node
*/
public Node chooseRandom(String scope) {
if (scope == null) {
scope = ROOT;
}
if (scope.startsWith(SCOPE_REVERSE_STR)) {
ArrayList<String> excludedScopes = new ArrayList();
excludedScopes.add(scope.substring(1));
return chooseRandom(ROOT, excludedScopes, null, null,
ANCESTOR_GENERATION_DEFAULT);
} else {
return chooseRandom(scope, null, null, null, ANCESTOR_GENERATION_DEFAULT);
}
}
/**
* Randomly choose a node in the scope, ano not in the exclude scope.
* @param scope range of nodes from which a node will be chosen. cannot start
* with ~
* @param excludedScopes the chosen node cannot be in these ranges. cannot
* starts with ~
* @return the chosen node
*/
public Node chooseRandom(String scope, List<String> excludedScopes) {
return chooseRandom(scope, excludedScopes, null, null,
ANCESTOR_GENERATION_DEFAULT);
}
/**
* Randomly choose a leaf node from <i>scope</i>.
*
* If scope starts with ~, choose one from the all nodes except for the
* ones in <i>scope</i>; otherwise, choose nodes from <i>scope</i>.
* If excludedNodes is given, choose a node that's not in excludedNodes.
*
* @param scope range of nodes from which a node will be chosen
* @param excludedNodes nodes to be excluded
*
* @return the chosen node
*/
public Node chooseRandom(String scope, Collection<Node> excludedNodes) {
if (scope == null) {
scope = ROOT;
}
if (scope.startsWith(SCOPE_REVERSE_STR)) {
ArrayList<String> excludedScopes = new ArrayList();
excludedScopes.add(scope.substring(1));
return chooseRandom(ROOT, excludedScopes, excludedNodes, null,
ANCESTOR_GENERATION_DEFAULT);
} else {
return chooseRandom(scope, null, excludedNodes, null,
ANCESTOR_GENERATION_DEFAULT);
}
}
/**
* Randomly choose a leaf node from <i>scope</i>.
*
* If scope starts with ~, choose one from the all nodes except for the
* ones in <i>scope</i>; otherwise, choose nodes from <i>scope</i>.
* If excludedNodes is given, choose a node that's not in excludedNodes.
*
* @param scope range of nodes from which a node will be chosen
* @param excludedNodes nodes to be excluded from.
* @param ancestorGen matters when excludeNodes is not null. It means the
* ancestor generation that's not allowed to share between chosen node and the
* excludedNodes. For example, if ancestorGen is 1, means chosen node
* cannot share the same parent with excludeNodes. If value is 2, cannot
* share the same grand parent, and so on. If ancestorGen is 0, then no
* effect.
*
* @return the chosen node
*/
public Node chooseRandom(String scope, Collection<Node> excludedNodes,
int ancestorGen) {
if (scope == null) {
scope = ROOT;
}
if (scope.startsWith(SCOPE_REVERSE_STR)) {
ArrayList<String> excludedScopes = new ArrayList();
excludedScopes.add(scope.substring(1));
return chooseRandom(ROOT, excludedScopes, excludedNodes, null,
ancestorGen);
} else {
return chooseRandom(scope, null, excludedNodes, null, ancestorGen);
}
}
/**
* Randomly choose one leaf node from <i>scope</i>, share the same generation
* ancestor with <i>affinityNode</i>, and exclude nodes in
* <i>excludeScope</i> and <i>excludeNodes</i>.
*
* @param scope range of nodes from which a node will be chosen, cannot start
* with ~
* @param excludedScopes ranges of nodes to be excluded, cannot start with ~
* @param excludedNodes nodes to be excluded
* @param affinityNode when not null, the chosen node should share the same
* ancestor with this node at generation ancestorGen.
* Ignored when value is null
* @param ancestorGen If 0, then no same generation ancestor enforcement on
* both excludedNodes and affinityNode. If greater than 0,
* then apply to affinityNode(if not null), or apply to
* excludedNodes if affinityNode is null
* @return the chosen node
*/
public Node chooseRandom(String scope, List<String> excludedScopes,
Collection<Node> excludedNodes, Node affinityNode, int ancestorGen) {
if (scope == null) {
scope = ROOT;
}
checkScope(scope);
checkExcludedScopes(excludedScopes);
checkAffinityNode(affinityNode);
checkAncestorGen(ancestorGen);
netlock.readLock().lock();
try {
return chooseNodeInternal(scope, -1, excludedScopes,
excludedNodes, affinityNode, ancestorGen);
} finally {
netlock.readLock().unlock();
}
}
/**
* Choose the leaf node at index <i>index</i> from <i>scope</i>, share the
* same generation ancestor with <i>affinityNode</i>, and exclude nodes in
* <i>excludeScope</i> and <i>excludeNodes</i>.
*
* @param leafIndex node index, exclude nodes in excludedScope and
* excludedNodes
* @param scope range of nodes from which a node will be chosen, cannot start
* with ~
* @param excludedScopes ranges of nodes to be excluded, cannot start with ~
* @param excludedNodes nodes to be excluded
* @param affinityNode when not null, the chosen node should share the same
* ancestor with this node at generation ancestorGen.
* Ignored when value is null
* @param ancestorGen If 0, then no same generation ancestor enforcement on
* both excludedNodes and affinityNode. If greater than 0,
* then apply to affinityNode(if not null), or apply to
* excludedNodes if affinityNode is null
* @return the chosen node
* Example:
*
* / --- root
* / \
* / \
* / \
* / \
* dc1 dc2
* / \ / \
* / \ / \
* / \ / \
* rack1 rack2 rack1 rack2
* / \ / \ / \ / \
* n1 n2 n3 n4 n5 n6 n7 n8
*
* Input:
* leafIndex = 1
* excludedScope = /dc2
* excludedNodes = {/dc1/rack1/n1}
* affinityNode = /dc1/rack2/n2
* ancestorGen = 2
*
* Output:
* node /dc1/rack2/n4
*
* Explanation:
* With affinityNode n2 and ancestorGen 2, it means we can only pick node
* from subtree /dc1. LeafIndex 1, so we pick the 2nd available node n4.
*
*/
public Node getNode(int leafIndex, String scope, List<String> excludedScopes,
Collection<Node> excludedNodes, Node affinityNode, int ancestorGen) {
Preconditions.checkArgument(leafIndex >= 0);
if (scope == null) {
scope = ROOT;
}
checkScope(scope);
checkExcludedScopes(excludedScopes);
checkAffinityNode(affinityNode);
checkAncestorGen(ancestorGen);
netlock.readLock().lock();
try {
return chooseNodeInternal(scope, leafIndex, excludedScopes,
excludedNodes, affinityNode, ancestorGen);
} finally {
netlock.readLock().unlock();
}
}
private Node chooseNodeInternal(String scope, int leafIndex,
List<String> excludedScopes, Collection<Node> excludedNodes,
Node affinityNode, int ancestorGen) {
Preconditions.checkArgument(scope != null);
String finalScope = scope;
if (affinityNode != null && ancestorGen > 0) {
Node affinityAncestor = affinityNode.getAncestor(ancestorGen);
if (affinityAncestor == null) {
throw new IllegalArgumentException("affinityNode " +
affinityNode.getNetworkFullPath() + " doesn't have ancestor on" +
" generation " + ancestorGen);
}
// affinity ancestor should has overlap with scope
if (affinityAncestor.getNetworkFullPath().startsWith(scope)){
finalScope = affinityAncestor.getNetworkFullPath();
} else if (!scope.startsWith(affinityAncestor.getNetworkFullPath())) {
return null;
}
// reset ancestor generation since the new scope is identified now
ancestorGen = 0;
}
// check overlap of excludedScopes and finalScope
List<String> mutableExcludedScopes = null;
if (excludedScopes != null && !excludedScopes.isEmpty()) {
mutableExcludedScopes = new ArrayList<>();
for (String s: excludedScopes) {
// excludeScope covers finalScope
if (finalScope.startsWith(s)) {
return null;
}
// excludeScope and finalScope share nothing case
if (s.startsWith(finalScope)) {
if (!mutableExcludedScopes.stream().anyMatch(
e -> s.startsWith(e))) {
mutableExcludedScopes.add(s);
}
}
}
}
// clone excludedNodes before remove duplicate in it
Collection<Node> mutableExNodes = null;
// Remove duplicate in excludedNodes
if (excludedNodes != null) {
mutableExNodes =
excludedNodes.stream().distinct().collect(Collectors.toList());
}
// remove duplicate in mutableExNodes and mutableExcludedScopes
NetUtils.removeDuplicate(this, mutableExNodes, mutableExcludedScopes,
ancestorGen);
// calculate available node count
Node scopeNode = getNode(finalScope);
int availableNodes = getAvailableNodesCount(
scopeNode.getNetworkFullPath(), mutableExcludedScopes, mutableExNodes,
ancestorGen);
if (availableNodes <= 0) {
LOG.warn("No available node in (scope=\"{}\" excludedScope=\"{}\" " +
"excludedNodes=\"{}\" ancestorGen=\"{}\").",
scopeNode.getNetworkFullPath(), excludedScopes, excludedNodes,
ancestorGen);
return null;
}
// scope is a Leaf node
if (!(scopeNode instanceof InnerNode)) {
return scopeNode;
}
Node ret;
int nodeIndex;
if (leafIndex >= 0) {
nodeIndex = leafIndex % availableNodes;
ret = ((InnerNode)scopeNode).getLeaf(nodeIndex, mutableExcludedScopes,
mutableExNodes, ancestorGen);
} else {
nodeIndex = ThreadLocalRandom.current().nextInt(availableNodes);
ret = ((InnerNode)scopeNode).getLeaf(nodeIndex, mutableExcludedScopes,
mutableExNodes, ancestorGen);
}
if (LOG.isDebugEnabled()) {
LOG.debug("Choosing node[index={},random={}] from \"{}\" available " +
"nodes, scope=\"{}\", excludedScope=\"{}\", excludeNodes=\"{}\".",
nodeIndex, (leafIndex == -1 ? "true" : "false"), availableNodes,
scopeNode.getNetworkFullPath(), excludedScopes, excludedNodes);
LOG.debug("Chosen node = {}", (ret == null ? "not found" :
ret.toString()));
}
return ret;
}
/** Return the distance cost between two nodes
* The distance cost from one node to its parent is it's parent's cost
* The distance cost between two nodes is calculated by summing up their
* distances cost to their closest common ancestor.
* @param node1 one node
* @param node2 another node
* @return the distance cost between node1 and node2 which is zero if they
* are the same or {@link Integer#MAX_VALUE} if node1 or node2 do not belong
* to the cluster
*/
public int getDistanceCost(Node node1, Node node2) {
if ((node1 != null && node2 != null && node1.equals(node2)) ||
(node1 == null && node2 == null)) {
return 0;
}
if (node1 == null || node2 == null) {
LOG.warn("One of the nodes is a null pointer");
return Integer.MAX_VALUE;
}
int cost = 0;
netlock.readLock().lock();
try {
if ((node1.getAncestor(maxLevel - 1) != clusterTree) ||
(node2.getAncestor(maxLevel - 1) != clusterTree)) {
LOG.debug("One of the nodes is outside of network topology");
return Integer.MAX_VALUE;
}
int level1 = node1.getLevel();
int level2 = node2.getLevel();
if (level1 > maxLevel || level2 > maxLevel) {
return Integer.MAX_VALUE;
}
while(level1 > level2 && node1 != null) {
node1 = node1.getParent();
level1--;
cost += node1 == null? 0 : node1.getCost();
}
while(level2 > level1 && node2 != null) {
node2 = node2.getParent();
level2--;
cost += node2 == null? 0 : node2.getCost();
}
while(node1 != null && node2 != null && node1 != node2) {
node1 = node1.getParent();
node2 = node2.getParent();
cost += node1 == null? 0 : node1.getCost();
cost += node2 == null? 0 : node2.getCost();
}
return cost;
} finally {
netlock.readLock().unlock();
}
}
/**
* Sort nodes array by network distance to <i>reader</i> to reduces network
* traffic and improves performance.
*
* As an additional twist, we also randomize the nodes at each network
* distance. This helps with load balancing when there is data skew.
*
* @param reader Node where need the data
* @param nodes Available replicas with the requested data
* @param activeLen Number of active nodes at the front of the array
*/
public List<? extends Node> sortByDistanceCost(Node reader,
List<? extends Node> nodes, int activeLen) {
/** Sort weights for the nodes array */
if (reader == null) {
return nodes;
}
int[] costs = new int[activeLen];
for (int i = 0; i < activeLen; i++) {
costs[i] = getDistanceCost(reader, nodes.get(i));
}
// Add cost/node pairs to a TreeMap to sort
TreeMap<Integer, List<Node>> tree = new TreeMap<Integer, List<Node>>();
for (int i = 0; i < activeLen; i++) {
int cost = costs[i];
Node node = nodes.get(i);
List<Node> list = tree.get(cost);
if (list == null) {
list = Lists.newArrayListWithExpectedSize(1);
tree.put(cost, list);
}
list.add(node);
}
List<Node> ret = new ArrayList<>();
for (List<Node> list: tree.values()) {
if (list != null) {
Collections.shuffle(list);
for (Node n: list) {
ret.add(n);
}
}
}
Preconditions.checkState(ret.size() == activeLen,
"Wrong number of nodes sorted!");
return ret;
}
/**
* Return the number of leaves in <i>scope</i> but not in
* <i>excludedNodes</i> and <i>excludeScope</i>.
* @param scope the scope
* @param excludedScopes excluded scopes
* @param mutableExcludedNodes a list of excluded nodes, content might be
* changed after the call
* @param ancestorGen same generation ancestor prohibit on excludedNodes
* @return number of available nodes
*/
private int getAvailableNodesCount(String scope, List<String> excludedScopes,
Collection<Node> mutableExcludedNodes, int ancestorGen) {
Preconditions.checkArgument(scope != null);
Node scopeNode = getNode(scope);
if (scopeNode == null) {
return 0;
}
NetUtils.removeOutscope(mutableExcludedNodes, scope);
List<Node> excludedAncestorList =
NetUtils.getAncestorList(this, mutableExcludedNodes, ancestorGen);
for (Node ancestor : excludedAncestorList) {
if (scope.startsWith(ancestor.getNetworkFullPath())){
return 0;
}
}
// number of nodes to exclude
int excludedCount = 0;
if (excludedScopes != null) {
for (String excludedScope: excludedScopes) {
Node excludedScopeNode = getNode(excludedScope);
if (excludedScopeNode != null) {
if (excludedScope.startsWith(scope)) {
excludedCount += excludedScopeNode.getNumOfLeaves();
} else if (scope.startsWith(excludedScope)) {
return 0;
}
}
}
}
// excludedNodes is not null case
if (mutableExcludedNodes != null && (!mutableExcludedNodes.isEmpty())) {
if (ancestorGen == 0) {
for (Node node: mutableExcludedNodes) {
if (contains(node)) {
excludedCount++;
}
}
} else {
for (Node ancestor : excludedAncestorList) {
if (ancestor.getNetworkFullPath().startsWith(scope)) {
excludedCount += ancestor.getNumOfLeaves();
}
}
}
}
int availableCount = scopeNode.getNumOfLeaves() - excludedCount;
Preconditions.checkState(availableCount >= 0);
return availableCount;
}
@Override
public String toString() {
// print max level
StringBuilder tree = new StringBuilder();
tree.append("Level: ");
tree.append(maxLevel);
tree.append("\n");
netlock.readLock().lock();
try {
// print the number of leaves
int numOfLeaves = clusterTree.getNumOfLeaves();
tree.append("Number of leaves:");
tree.append(numOfLeaves);
tree.append("\n");
// print all nodes
for (int i = 0; i < numOfLeaves; i++) {
tree.append(clusterTree.getLeaf(i).getNetworkFullPath());
tree.append("\n");
}
} finally {
netlock.readLock().unlock();
}
return tree.toString();
}
private void checkScope(String scope) {
if (scope != null && scope.startsWith(SCOPE_REVERSE_STR)) {
throw new IllegalArgumentException("scope " + scope +
" should not start with " + SCOPE_REVERSE_STR);
}
}
private void checkExcludedScopes(List<String> excludedScopes) {
if (!CollectionUtils.isEmpty(excludedScopes)) {
excludedScopes.stream().forEach(scope -> {
if (scope.startsWith(SCOPE_REVERSE_STR)) {
throw new IllegalArgumentException("excludedScope " + scope +
" cannot start with " + SCOPE_REVERSE_STR);
}
});
}
}
private void checkAffinityNode(Node affinityNode) {
if (affinityNode != null && (!contains(affinityNode))) {
throw new IllegalArgumentException("Affinity node " +
affinityNode.getNetworkFullPath() + " is not a member of topology");
}
}
private void checkAncestorGen(int ancestorGen) {
if (ancestorGen > (maxLevel - 1) || ancestorGen < 0) {
throw new IllegalArgumentException("ancestorGen " + ancestorGen +
" exceeds this network topology acceptable level [0, " +
(maxLevel - 1) + "]");
}
}
}

101
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/net/Node.java
View File

@ -1,101 +0,0 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdds.scm.net;
/**
* The interface defines a node in a network topology.
* A node may be a leave representing a data node or an inner
* node representing a data center or rack.
* Each node has a name and its location in the network is
* decided by a string with syntax similar to a file name.
* For example, a data node's name is hostname:port# and if it's located at
* rack "orange" in data center "dog", the string representation of its
* network location will be /dog/orange.
*/
public interface Node {
/** @return the string representation of this node's network location path,
* exclude itself. In another words, its parent's full network location */
String getNetworkLocation();
/**
* Set this node's network location.
* @param location it's network location
*/
void setNetworkLocation(String location);
/** @return this node's self name in network topology. This should be node's
* IP or hostname.
* */
String getNetworkName();
/**
* Set this node's name, can be hostname or Ipaddress.
* @param name it's network name
*/
void setNetworkName(String name);
/** @return this node's full path in network topology. It's the concatenation
* of location and name.
* */
String getNetworkFullPath();
/** @return this node's parent */
InnerNode getParent();
/**
* Set this node's parent.
* @param parent the parent
*/
void setParent(InnerNode parent);
/** @return this node's ancestor, generation 0 is itself, generation 1 is
* node's parent, and so on.*/
Node getAncestor(int generation);
/**
* @return this node's level in the tree.
* E.g. the root of a tree returns 1 and root's children return 2
*/
int getLevel();
/**
* Set this node's level in the tree.
* @param i the level
*/
void setLevel(int i);
/**
* @return this node's cost when network traffic go through it.
* E.g. the cost of going cross a switch is 1, and cost of going through a
* datacenter can be 5.
* Be default the cost of leaf datanode is 0, all other node is 1.
*/
int getCost();
/** @return the leaf nodes number under this node. */
int getNumOfLeaves();
/**
* Judge if this node is an ancestor of node <i>n</i>.
* Ancestor includes itself and parents case.
*
* @param n a node
* @return true if this node is an ancestor of <i>n</i>
*/
boolean isAncestor(Node n);
}

222
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/net/NodeImpl.java
View File

@ -1,222 +0,0 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdds.scm.net;
import com.google.common.base.Preconditions;
import static org.apache.hadoop.hdds.scm.net.NetConstants.ROOT;
import static org.apache.hadoop.hdds.scm.net.NetConstants.PATH_SEPARATOR_STR;
/**
* A thread safe class that implements interface Node.
*/
public class NodeImpl implements Node {
// host:port#
private String name;
// string representation of this node's location, such as /dc1/rack1
private String location;
// location + "/" + name
private String path;
// which level of the tree the node resides, start from 1 for root
private int level;
// node's parent
private InnerNode parent;
// the cost to go through this node
private final int cost;
/**
* Construct a node from its name and its location.
* @param name this node's name (can be null, must not contain
* {@link NetConstants#PATH_SEPARATOR})
* @param location this node's location
*/
public NodeImpl(String name, String location, int cost) {
if (name != null && name.contains(PATH_SEPARATOR_STR)) {
throw new IllegalArgumentException(
"Network location name:" + name + " should not contain " +
PATH_SEPARATOR_STR);
}
this.name = (name == null) ? ROOT : name;
this.location = NetUtils.normalize(location);
this.path = getPath();
this.cost = cost;
}
/**
* Construct a node from its name and its location.
*
* @param name this node's name (can be null, must not contain
* {@link NetConstants#PATH_SEPARATOR})
* @param location this node's location
* @param parent this node's parent node
* @param level this node's level in the tree
* @param cost this node's cost if traffic goes through it
*/
public NodeImpl(String name, String location, InnerNode parent, int level,
int cost) {
this(name, location, cost);
this.parent = parent;
this.level = level;
}
/**
* @return this node's name
*/
public String getNetworkName() {
return name;
}
/**
* Set this node's name, can be hostname or Ipaddress.
* @param networkName it's network name
*/
public void setNetworkName(String networkName) {
this.name = networkName;
this.path = getPath();
}
/**
* @return this node's network location
*/
public String getNetworkLocation() {
return location;
}
/**
* Set this node's network location.
* @param networkLocation it's network location
*/
@Override
public void setNetworkLocation(String networkLocation) {
this.location = networkLocation;
this.path = getPath();
}
/**
* @return this node's full path in network topology. It's the concatenation
* of location and name.
*/
public String getNetworkFullPath() {
return path;
}
/**
* @return this node's parent
*/
public InnerNode getParent() {
return parent;
}
/**
* @return this node's ancestor, generation 0 is itself, generation 1 is
* node's parent, and so on.
*/
public Node getAncestor(int generation) {
Preconditions.checkArgument(generation >= 0);
Node current = this;
while (generation > 0 && current != null) {
current = current.getParent();
generation--;
}
return current;
}
/**
* Set this node's parent.
*
* @param parent the parent
*/
public void setParent(InnerNode parent) {
this.parent = parent;
}
/**
* @return this node's level in the tree.
* E.g. the root of a tree returns 0 and its children return 1
*/
public int getLevel() {
return this.level;
}
/**
* Set this node's level in the tree.
*
* @param level the level
*/
public void setLevel(int level) {
this.level = level;
}
/**
* @return this node's cost when network traffic go through it.
* E.g. the cost of going cross a switch is 1, and cost of going through a
* datacenter is 5.
* Be default the cost of leaf datanode is 0, all other inner node is 1.
*/
public int getCost() {
return this.cost;
}
/** @return the leaf nodes number under this node. */
public int getNumOfLeaves() {
return 1;
}
/**
* Check if this node is an ancestor of node <i>node</i>. Ancestor includes
* itself and parents case;
* @param node a node
* @return true if this node is an ancestor of <i>node</i>
*/
public boolean isAncestor(Node node) {
return this.getNetworkFullPath().equals(PATH_SEPARATOR_STR) ||
node.getNetworkLocation().startsWith(this.getNetworkFullPath()) ||
node.getNetworkFullPath().equalsIgnoreCase(
this.getNetworkFullPath());
}
@Override
public boolean equals(Object to) {
if (to == null) {
return false;
}
if (this == to) {
return true;
}
return this.toString().equals(to.toString());
}
@Override
public int hashCode() {
return toString().hashCode();
}
/**
* @return this node's path as its string representation
*/
@Override
public String toString() {
return getNetworkFullPath();
}
private String getPath() {
return this.location.equals(PATH_SEPARATOR_STR) ?
this.location + this.name :
this.location + PATH_SEPARATOR_STR + this.name;
}
}

183
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/net/NodeSchema.java
View File

@ -1,183 +0,0 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdds.scm.net;
import org.apache.hadoop.HadoopIllegalArgumentException;
import java.util.List;
/**
* Network topology schema to housekeeper relevant information.
*/
public final class NodeSchema {
/**
* Network topology layer type enum definition.
*/
public enum LayerType{
ROOT("Root", NetConstants.INNER_NODE_COST_DEFAULT),
INNER_NODE("InnerNode", NetConstants.INNER_NODE_COST_DEFAULT),
LEAF_NODE("Leaf", NetConstants.NODE_COST_DEFAULT);
private final String description;
// default cost
private final int cost;
LayerType(String description, int cost) {
this.description = description;
this.cost = cost;
}
@Override
public String toString() {
return description;
}
public int getCost(){
return cost;
}
public static LayerType getType(String typeStr) {
for (LayerType type: LayerType.values()) {
if (typeStr.equalsIgnoreCase(type.toString())) {
return type;
}
}
return null;
}
}
// default cost
private int cost;
// layer Type, mandatory property
private LayerType type;
// default name, can be null or ""
private String defaultName;
// layer prefix, can be null or ""
private String prefix;
// sublayer
private List<NodeSchema> sublayer;
/**
* Builder for NodeSchema.
*/
public static class Builder {
private int cost = -1;
private LayerType type;
private String defaultName;
private String prefix;
public Builder setCost(int nodeCost) {
this.cost = nodeCost;
return this;
}
public Builder setPrefix(String nodePrefix) {
this.prefix = nodePrefix;
return this;
}
public Builder setType(LayerType nodeType) {
this.type = nodeType;
return this;
}
public Builder setDefaultName(String nodeDefaultName) {
this.defaultName = nodeDefaultName;
return this;
}
public NodeSchema build() {
if (type == null) {
throw new HadoopIllegalArgumentException("Type is mandatory for a " +
"network topology node layer definition");
}
if (cost == -1) {
cost = type.getCost();
}
return new NodeSchema(type, cost, prefix, defaultName);
}
}
/**
* Constructor.
* @param type layer type
* @param cost layer's default cost
* @param prefix layer's prefix
* @param defaultName layer's default name is if specified
*/
public NodeSchema(LayerType type, int cost, String prefix,
String defaultName) {
this.type = type;
this.cost = cost;
this.prefix = prefix;
this.defaultName = defaultName;
}
/**
* Constructor. This constructor is only used when build NodeSchema from
* YAML file.
*/
public NodeSchema() {
this.type = LayerType.INNER_NODE;
}
public boolean matchPrefix(String name) {
if (name == null || name.isEmpty() || prefix == null || prefix.isEmpty()) {
return false;
}
return name.trim().toLowerCase().startsWith(prefix.toLowerCase());
}
public LayerType getType() {
return this.type;
}
public void setType(LayerType type) {
this.type = type;
}
public String getPrefix() {
return this.prefix;
}
public void setPrefix(String prefix) {
this.prefix = prefix;
}
public String getDefaultName() {
return this.defaultName;
}
public void setDefaultName(String name) {
this.defaultName = name;
}
public int getCost() {
return this.cost;
}
public void setCost(int cost) {
this.cost = cost;
}
public void setSublayer(List<NodeSchema> sublayer) {
this.sublayer = sublayer;
}
public List<NodeSchema> getSublayer() {
return sublayer;
}
}

489
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/net/NodeSchemaLoader.java
View File

@ -1,489 +0,0 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdds.scm.net;
import org.apache.commons.io.FilenameUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import org.w3c.dom.Text;
import org.xml.sax.SAXException;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.hadoop.hdds.scm.net.NodeSchema.LayerType;
import org.yaml.snakeyaml.Yaml;
/**
* A Network topology layer schema loading tool that loads user defined network
* layer schema data from a XML configuration file.
*/
public final class NodeSchemaLoader {
private static final Logger LOG
= LoggerFactory.getLogger(NodeSchemaLoader.class);
private static final String CONFIGURATION_TAG = "configuration";
private static final String LAYOUT_VERSION_TAG = "layoutversion";
private static final String TOPOLOGY_TAG = "topology";
private static final String TOPOLOGY_PATH = "path";
private static final String TOPOLOGY_ENFORCE_PREFIX = "enforceprefix";
private static final String LAYERS_TAG = "layers";
private static final String LAYER_TAG = "layer";
private static final String LAYER_ID = "id";
private static final String LAYER_TYPE = "type";
private static final String LAYER_COST = "cost";
private static final String LAYER_PREFIX = "prefix";
private static final String LAYER_DEFAULT_NAME = "default";
private static final int LAYOUT_VERSION = 1;
private volatile static NodeSchemaLoader instance = null;
private NodeSchemaLoader() {}
public static NodeSchemaLoader getInstance() {
if (instance == null) {
instance = new NodeSchemaLoader();
}
return instance;
}
/**
* Class to house keep the result of parsing a network topology schema file.
*/
public static class NodeSchemaLoadResult {
private List<NodeSchema> schemaList;
private boolean enforcePrefix;
NodeSchemaLoadResult(List<NodeSchema> schemaList, boolean enforcePrefix) {
this.schemaList = schemaList;
this.enforcePrefix = enforcePrefix;
}
public boolean isEnforePrefix() {
return enforcePrefix;
}
public List<NodeSchema> getSchemaList() {
return schemaList;
}
}
/**
* Load user defined network layer schemas from a XML/YAML configuration file.
* @param schemaFilePath path of schema file
* @return all valid node schemas defined in schema file
*/
public NodeSchemaLoadResult loadSchemaFromFile(String schemaFilePath)
throws IllegalArgumentException, FileNotFoundException {
try {
File schemaFile = new File(schemaFilePath);
if (schemaFile.exists()) {
LOG.info("Load network topology schema file " +
schemaFile.getAbsolutePath());
try (FileInputStream inputStream = new FileInputStream(schemaFile)) {
return loadSchemaFromStream(schemaFilePath, inputStream);
}
} else {
// try to load with classloader
ClassLoader classloader =
Thread.currentThread().getContextClassLoader();
if (classloader == null) {
classloader = NodeSchemaLoader.class.getClassLoader();
}
if (classloader != null) {
try (InputStream stream = classloader
.getResourceAsStream(schemaFilePath)) {
if (stream != null) {
LOG.info("Loading file from " + classloader
.getResources(schemaFilePath));
return loadSchemaFromStream(schemaFilePath, stream);
}
}
}
}
String msg = "Network topology layer schema file " +
schemaFilePath + "[" + schemaFile.getAbsolutePath() +
"] is not found.";
LOG.warn(msg);
throw new FileNotFoundException(msg);
} catch (FileNotFoundException e) {
throw e;
} catch (ParserConfigurationException | IOException | SAXException e) {
throw new IllegalArgumentException("Failed to load network topology node"
+ " schema file: " + schemaFilePath + " , error:" + e.getMessage(),
e);
}
}
private NodeSchemaLoadResult loadSchemaFromStream(String schemaFilePath,
InputStream stream)
throws ParserConfigurationException, SAXException, IOException {
if (FilenameUtils.getExtension(schemaFilePath).toLowerCase()
.compareTo("yaml") == 0) {
return loadSchemaFromYaml(stream);
} else {
return loadSchema(stream);
}
}
/**
* Load network topology layer schemas from a XML configuration file.
* @param inputStream schema file as an inputStream
* @return all valid node schemas defined in schema file
* @throws ParserConfigurationException ParserConfigurationException happen
* @throws IOException no such schema file
* @throws SAXException xml file has some invalid elements
* @throws IllegalArgumentException xml file content is logically invalid
*/
private NodeSchemaLoadResult loadSchema(InputStream inputStream) throws
ParserConfigurationException, SAXException, IOException {
LOG.info("Loading network topology layer schema file");
// Read and parse the schema file.
DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
dbf.setIgnoringComments(true);
DocumentBuilder builder = dbf.newDocumentBuilder();
Document doc = builder.parse(inputStream);
Element root = doc.getDocumentElement();
if (!CONFIGURATION_TAG.equals(root.getTagName())) {
throw new IllegalArgumentException("Bad network topology layer schema " +
"configuration file: top-level element not <" + CONFIGURATION_TAG +
">");
}
NodeSchemaLoadResult schemaList;
if (root.getElementsByTagName(LAYOUT_VERSION_TAG).getLength() == 1) {
if (loadLayoutVersion(root) == LAYOUT_VERSION) {
if (root.getElementsByTagName(LAYERS_TAG).getLength() == 1) {
Map<String, NodeSchema> schemas = loadLayersSection(root);
if (root.getElementsByTagName(TOPOLOGY_TAG).getLength() == 1) {
schemaList = loadTopologySection(root, schemas);
} else {
throw new IllegalArgumentException("Bad network topology layer " +
"schema configuration file: no or multiple <" + TOPOLOGY_TAG +
"> element");
}
} else {
throw new IllegalArgumentException("Bad network topology layer schema"
+ " configuration file: no or multiple <" + LAYERS_TAG +
">element");
}
} else {
throw new IllegalArgumentException("The parse failed because of bad "
+ LAYOUT_VERSION_TAG + " value, expected:" + LAYOUT_VERSION);
}
} else {
throw new IllegalArgumentException("Bad network topology layer schema " +
"configuration file: no or multiple <" + LAYOUT_VERSION_TAG +
"> elements");
}
return schemaList;
}
/**
* Load network topology layer schemas from a YAML configuration file.
* @param schemaFile as inputStream
* @return all valid node schemas defined in schema file
* @throws ParserConfigurationException ParserConfigurationException happen
* @throws IOException no such schema file
* @throws SAXException xml file has some invalid elements
* @throws IllegalArgumentException xml file content is logically invalid
*/
private NodeSchemaLoadResult loadSchemaFromYaml(InputStream schemaFile) {
LOG.info("Loading network topology layer schema file {}", schemaFile);
NodeSchemaLoadResult finalSchema;
try {
Yaml yaml = new Yaml();
NodeSchema nodeTree;
nodeTree = yaml.loadAs(schemaFile, NodeSchema.class);
List<NodeSchema> schemaList = new ArrayList<>();
if (nodeTree.getType() != LayerType.ROOT) {
throw new IllegalArgumentException("First layer is not a ROOT node."
+ " schema file.");
}
schemaList.add(nodeTree);
if (nodeTree.getSublayer() != null) {
nodeTree = nodeTree.getSublayer().get(0);
}
while (nodeTree != null) {
if (nodeTree.getType() == LayerType.LEAF_NODE
&& nodeTree.getSublayer() != null) {
throw new IllegalArgumentException("Leaf node in the middle of path."
+ " schema file.");
}
if (nodeTree.getType() == LayerType.ROOT) {
throw new IllegalArgumentException("Multiple root nodes are defined."
+ " schema file.");
}
schemaList.add(nodeTree);
if (nodeTree.getSublayer() != null) {
nodeTree = nodeTree.getSublayer().get(0);
} else {
break;
}
}
finalSchema = new NodeSchemaLoadResult(schemaList, true);
} catch (Exception e) {
throw new IllegalArgumentException("Fail to load network topology node"
+ " schema file: " + schemaFile + " , error:"
+ e.getMessage(), e);
}
return finalSchema;
}
/**
* Load layoutVersion from root element in the XML configuration file.
* @param root root element
* @return layout version
*/
private int loadLayoutVersion(Element root) {
int layoutVersion;
Text text = (Text) root.getElementsByTagName(LAYOUT_VERSION_TAG)
.item(0).getFirstChild();
if (text != null) {
String value = text.getData().trim();
try {
layoutVersion = Integer.parseInt(value);
} catch (NumberFormatException e) {
throw new IllegalArgumentException("Bad " + LAYOUT_VERSION_TAG +
" value " + value + " is found. It should be an integer.");
}
} else {
throw new IllegalArgumentException("Value of <" + LAYOUT_VERSION_TAG +
"> is null");
}
return layoutVersion;
}
/**
* Load layers from root element in the XML configuration file.
* @param root root element
* @return A map of node schemas with layer ID and layer schema
*/
private Map<String, NodeSchema> loadLayersSection(Element root) {
NodeList elements = root.getElementsByTagName(LAYER_TAG);
Map<String, NodeSchema> schemas = new HashMap<String, NodeSchema>();
for (int i = 0; i < elements.getLength(); i++) {
Node node = elements.item(i);
if (node instanceof Element) {
Element element = (Element) node;
if (LAYER_TAG.equals(element.getTagName())) {
String layerId = element.getAttribute(LAYER_ID);
NodeSchema schema = parseLayerElement(element);
if (!schemas.containsValue(schema)) {
schemas.put(layerId, schema);
} else {
throw new IllegalArgumentException("Repetitive layer in network " +
"topology node schema configuration file: " + layerId);
}
} else {
throw new IllegalArgumentException("Bad element in network topology "
+ "node schema configuration file: " + element.getTagName());
}
}
}
// Integrity check, only one ROOT and one LEAF is allowed
boolean foundRoot = false;
boolean foundLeaf = false;
for(NodeSchema schema: schemas.values()) {
if (schema.getType() == LayerType.ROOT) {
if (foundRoot) {
throw new IllegalArgumentException("Multiple ROOT layers are found" +
" in network topology schema configuration file");
} else {
foundRoot = true;
}
}
if (schema.getType() == LayerType.LEAF_NODE) {
if (foundLeaf) {
throw new IllegalArgumentException("Multiple LEAF layers are found" +
" in network topology schema configuration file");
} else {
foundLeaf = true;
}
}
}
if (!foundRoot) {
throw new IllegalArgumentException("No ROOT layer is found" +
" in network topology schema configuration file");
}
if (!foundLeaf) {
throw new IllegalArgumentException("No LEAF layer is found" +
" in network topology schema configuration file");
}
return schemas;
}
/**
* Load network topology from root element in the XML configuration file and
* sort node schemas according to the topology path.
* @param root root element
* @param schemas schema map
* @return all valid node schemas defined in schema file
*/
private NodeSchemaLoadResult loadTopologySection(Element root,
Map<String, NodeSchema> schemas) {
NodeList elements = root.getElementsByTagName(TOPOLOGY_TAG)
.item(0).getChildNodes();
List<NodeSchema> schemaList = new ArrayList<NodeSchema>();
boolean enforecePrefix = false;
for (int i = 0; i < elements.getLength(); i++) {
Node node = elements.item(i);
if (node instanceof Element) {
Element element = (Element) node;
String tagName = element.getTagName();
// Get the nonnull text value.
Text text = (Text) element.getFirstChild();
String value;
if (text != null) {
value = text.getData().trim();
if (value.isEmpty()) {
// Element with empty value is ignored
continue;
}
} else {
throw new IllegalArgumentException("Value of <" + tagName
+ "> is null");
}
if (TOPOLOGY_PATH.equals(tagName)) {
if(value.startsWith(NetConstants.PATH_SEPARATOR_STR)) {
value = value.substring(1, value.length());
}
String[] layerIDs = value.split(NetConstants.PATH_SEPARATOR_STR);
if (layerIDs == null || layerIDs.length != schemas.size()) {
throw new IllegalArgumentException("Topology path depth doesn't "
+ "match layer element numbers");
}
for (int j = 0; j < layerIDs.length; j++) {
if (schemas.get(layerIDs[j]) == null) {
throw new IllegalArgumentException("No layer found for id " +
layerIDs[j]);
}
}
if (schemas.get(layerIDs[0]).getType() != LayerType.ROOT) {
throw new IllegalArgumentException("Topology path doesn't start "
+ "with ROOT layer");
}
if (schemas.get(layerIDs[layerIDs.length -1]).getType() !=
LayerType.LEAF_NODE) {
throw new IllegalArgumentException("Topology path doesn't end "
+ "with LEAF layer");
}
for (int j = 0; j < layerIDs.length; j++) {
schemaList.add(schemas.get(layerIDs[j]));
}
} else if (TOPOLOGY_ENFORCE_PREFIX.equalsIgnoreCase(tagName)) {
enforecePrefix = Boolean.parseBoolean(value);
} else {
throw new IllegalArgumentException("Unsupported Element <" +
tagName + ">");
}
}
}
// Integrity check
if (enforecePrefix) {
// Every InnerNode should have prefix defined
for (NodeSchema schema: schemas.values()) {
if (schema.getType() == LayerType.INNER_NODE &&
schema.getPrefix() == null) {
throw new IllegalArgumentException("There is layer without prefix " +
"defined while prefix is enforced.");
}
}
}
return new NodeSchemaLoadResult(schemaList, enforecePrefix);
}
/**
* Load a layer from a layer element in the XML configuration file.
* @param element network topology node layer element
* @return ECSchema
*/
private NodeSchema parseLayerElement(Element element) {
NodeList fields = element.getChildNodes();
LayerType type = null;
int cost = 0;
String prefix = null;
String defaultName = null;
for (int i = 0; i < fields.getLength(); i++) {
Node fieldNode = fields.item(i);
if (fieldNode instanceof Element) {
Element field = (Element) fieldNode;
String tagName = field.getTagName();
// Get the nonnull text value.
Text text = (Text) field.getFirstChild();
String value;
if (text != null) {
value = text.getData().trim();
if (value.isEmpty()) {
// Element with empty value is ignored
continue;
}
} else {
continue;
}
if (LAYER_COST.equalsIgnoreCase(tagName)) {
cost = Integer.parseInt(value);
if (cost < 0) {
throw new IllegalArgumentException(
"Cost should be positive number or 0");
}
} else if (LAYER_TYPE.equalsIgnoreCase(tagName)) {
type = NodeSchema.LayerType.getType(value);
if (type == null) {
throw new IllegalArgumentException(
"Unsupported layer type:" + value);
}
} else if (LAYER_PREFIX.equalsIgnoreCase(tagName)) {
prefix = value;
} else if (LAYER_DEFAULT_NAME.equalsIgnoreCase(tagName)) {
defaultName = value;
} else {
throw new IllegalArgumentException("Unsupported Element <" + tagName
+ ">");
}
}
}
// type is a mandatory property
if (type == null) {
throw new IllegalArgumentException("Missing type Element");
}
return new NodeSchema(type, cost, prefix, defaultName);
}
}

135
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/net/NodeSchemaManager.java
View File

@ -1,135 +0,0 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdds.scm.net;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.scm.net.NodeSchemaLoader.NodeSchemaLoadResult;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.conf.Configuration;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
/** The class manages all network topology schemas. */
public final class NodeSchemaManager {
private static final Logger LOG = LoggerFactory.getLogger(
NodeSchemaManager.class);
// All schema saved and sorted from ROOT to LEAF node
private List<NodeSchema> allSchema;
// enforcePrefix only applies to INNER_NODE
private boolean enforcePrefix;
// max level, includes ROOT level
private int maxLevel = -1;
private volatile static NodeSchemaManager instance = null;
private NodeSchemaManager() {
}
public static NodeSchemaManager getInstance() {
if (instance == null) {
instance = new NodeSchemaManager();
}
return instance;
}
public void init(Configuration conf) {
/**
* Load schemas from network topology schema configuration file
*/
String schemaFile = conf.get(
ScmConfigKeys.OZONE_SCM_NETWORK_TOPOLOGY_SCHEMA_FILE,
ScmConfigKeys.OZONE_SCM_NETWORK_TOPOLOGY_SCHEMA_FILE_DEFAULT);
NodeSchemaLoadResult result;
try {
result = NodeSchemaLoader.getInstance().loadSchemaFromFile(schemaFile);
allSchema = result.getSchemaList();
enforcePrefix = result.isEnforePrefix();
maxLevel = allSchema.size();
} catch (Throwable e) {
String msg = "Failed to load schema file:" + schemaFile
+ ", error: " + e.getMessage();
LOG.error(msg, e);
throw new RuntimeException(msg, e);
}
}
@VisibleForTesting
public void init(NodeSchema[] schemas, boolean enforce) {
allSchema = new ArrayList<>();
allSchema.addAll(Arrays.asList(schemas));
enforcePrefix = enforce;
maxLevel = schemas.length;
}
public int getMaxLevel() {
return maxLevel;
}
public int getCost(int level) {
Preconditions.checkArgument(level <= maxLevel &&
level >= (NetConstants.ROOT_LEVEL));
return allSchema.get(level - NetConstants.ROOT_LEVEL).getCost();
}
/**
* Given a incomplete network path, return its complete network path if
* possible. E.g. input is 'node1', output is '/rack-default/node1' if this
* schema manages ROOT, RACK and LEAF, with prefix defined and enforce prefix
* enabled.
*
* @param path the incomplete input path
* @return complete path, null if cannot carry out complete action or action
* failed
*/
public String complete(String path) {
if (!enforcePrefix) {
return null;
}
String normalizedPath = NetUtils.normalize(path);
String[] subPath = normalizedPath.split(NetConstants.PATH_SEPARATOR_STR);
if ((subPath.length) == maxLevel) {
return path;
}
StringBuffer newPath = new StringBuffer(NetConstants.ROOT);
// skip the ROOT and LEAF layer
int i, j;
for (i = 1, j = 1; i < subPath.length && j < (allSchema.size() - 1);) {
if (allSchema.get(j).matchPrefix(subPath[i])) {
newPath.append(NetConstants.PATH_SEPARATOR_STR + subPath[i]);
i++;
j++;
} else {
newPath.append(allSchema.get(j).getDefaultName());
j++;
}
}
if (i == (subPath.length - 1)) {
newPath.append(NetConstants.PATH_SEPARATOR_STR + subPath[i]);
return newPath.toString();
}
return null;
}
}

21
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/net/package-info.java
View File

@ -1,21 +0,0 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdds.scm.net;
/**
The network topology supported by Ozone.
*/

Some files were not shown because too many files have changed in this diff Show More