big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-9375. Use Configured in GpuDiscoverer and FpgaDiscoverer (#1131)

Browse Source 
Contributed by Adam Antal
This commit is contained in:
Adam Antal 2019-08-02 11:24:09 +02:00 committed by Szilard Nemeth
parent e872ceb810
commit 95fc38f2e9
2 changed files with 12 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/fpga/FpgaDiscoverer.java
View File

@ -28,6 +28,7 @@
import java.util.stream.Collectors;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.util.Shell.ShellCommandExecutor;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
@ -46,11 +47,10 @@
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Sets;
public class FpgaDiscoverer {
public class FpgaDiscoverer extends Configured {
private static final Logger LOG = LoggerFactory.getLogger(
FpgaDiscoverer.class);
private Configuration conf = null;
private AbstractFpgaVendorPlugin plugin = null;
private List<FpgaResourceAllocator.FpgaDevice> currentFpgaInfo = null;
@ -64,11 +64,6 @@ void setScriptRunner(Function<String, Optional<String>> scriptRunner) {
this.scriptRunner = scriptRunner;
}
@VisibleForTesting
public void setConf(Configuration configuration) {
this.conf = configuration;
}
public List<FpgaDevice> getCurrentFpgaInfo() {
return currentFpgaInfo;
}
@ -82,7 +77,7 @@ public boolean diagnose() {
}
public void initialize(Configuration config) throws YarnException {
this.conf = config;
setConf(config);
this.plugin.initPlugin(config);
// Try to diagnose FPGA
LOG.info("Trying to diagnose FPGA information ...");
@ -100,11 +95,11 @@ public void initialize(Configuration config) throws YarnException {
public List<FpgaDevice> discover()
throws ResourceHandlerException {
List<FpgaDevice> list;
String allowed = this.conf.get(YarnConfiguration.NM_FPGA_ALLOWED_DEVICES);
String allowed = getConf().get(YarnConfiguration.NM_FPGA_ALLOWED_DEVICES);
String availableDevices = conf.get(
String availableDevices = getConf().get(
YarnConfiguration.NM_FPGA_AVAILABLE_DEVICES);
String discoveryScript = conf.get(
String discoveryScript = getConf().get(
YarnConfiguration.NM_FPGA_DEVICE_DISCOVERY_SCRIPT);
FPGADiscoveryStrategy discoveryStrategy;

12
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuDiscoverer.java
View File

@ -25,6 +25,7 @@
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.util.Shell;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
@ -45,7 +46,7 @@
@InterfaceAudience.Private
@InterfaceStability.Unstable
public class GpuDiscoverer {
public class GpuDiscoverer extends Configured {
public static final Logger LOG = LoggerFactory.getLogger(
GpuDiscoverer.class);
@VisibleForTesting
@ -61,7 +62,6 @@ public class GpuDiscoverer {
private static final int MAX_EXEC_TIMEOUT_MS = 10 * 1000;
private static final int MAX_REPEATED_ERROR_ALLOWED = 10;
private Configuration conf = null;
private String pathOfGpuBinary = null;
private Map<String, String> environment = new HashMap<>();
@ -71,7 +71,7 @@ public class GpuDiscoverer {
private List<GpuDevice> gpuDevicesFromUser;
private void validateConfOrThrowException() throws YarnException {
if (conf == null) {
if (getConf() == null) {
throw new YarnException("Please initialize (call initialize) before use "
+ GpuDiscoverer.class.getSimpleName());
}
@ -144,7 +144,7 @@ synchronized GpuDeviceInformation getGpuDeviceInformation()
}
private boolean IsAutoDiscoveryEnabled() {
String allowedDevicesStr = conf.get(
String allowedDevicesStr = getConf().get(
YarnConfiguration.NM_GPU_ALLOWED_DEVICES,
YarnConfiguration.AUTOMATICALLY_DISCOVER_GPU_DEVICES);
return allowedDevicesStr.equals(
@ -205,7 +205,7 @@ private List<GpuDevice> parseGpuDevicesFromAutoDiscoveredGpuInfo()
*/
private List<GpuDevice> parseGpuDevicesFromUserDefinedValues()
throws YarnException {
String devices = conf.get(
String devices = getConf().get(
YarnConfiguration.NM_GPU_ALLOWED_DEVICES,
YarnConfiguration.AUTOMATICALLY_DISCOVER_GPU_DEVICES);
@ -249,7 +249,7 @@ private GpuDevice parseGpuDevice(String device, String[] splitByColon,
public synchronized void initialize(Configuration config)
throws YarnException {
this.conf = config;
setConf(config);
if (IsAutoDiscoveryEnabled()) {
numOfErrorExecutionSinceLastSucceed = 0;
lookUpAutoDiscoveryBinary(config);