big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-15013. Fix ResourceEstimator findbugs issues. (asuresh)

Browse Source
This commit is contained in:
Arun Suresh 2017-11-02 17:14:07 -07:00
parent e6ec02001f
commit 53c0fb7efe
4 changed files with 53 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

85
hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/service/ResourceEstimatorService.java
View File

@ -65,51 +65,49 @@
@Singleton @Path("/resourceestimator") public class ResourceEstimatorService {
private static final Logger LOGGER =
LoggerFactory.getLogger(ResourceEstimatorService.class);
private static SkylineStore skylineStore;
private static Solver solver;
private static LogParser logParser;
private static LogParserUtil logParserUtil = new LogParserUtil();
private static Configuration config;
private static Gson gson;
private static Type rleType;
private static Type skylineStoreType;
private final SkylineStore skylineStore;
private final Solver solver;
private final LogParser logParser;
private final LogParserUtil logParserUtil = new LogParserUtil();
private final Configuration config;
private final Gson gson;
private final Type rleType;
private final Type skylineStoreType;
public ResourceEstimatorService() throws ResourceEstimatorException {
if (skylineStore == null) {
try {
config = new Configuration();
config.addResource(ResourceEstimatorConfiguration.CONFIG_FILE);
skylineStore = ResourceEstimatorUtil.createProviderInstance(config,
ResourceEstimatorConfiguration.SKYLINESTORE_PROVIDER,
ResourceEstimatorConfiguration.DEFAULT_SKYLINESTORE_PROVIDER,
SkylineStore.class);
logParser = ResourceEstimatorUtil.createProviderInstance(config,
ResourceEstimatorConfiguration.TRANSLATOR_PROVIDER,
ResourceEstimatorConfiguration.DEFAULT_TRANSLATOR_PROVIDER,
LogParser.class);
logParser.init(config, skylineStore);
logParserUtil.setLogParser(logParser);
solver = ResourceEstimatorUtil.createProviderInstance(config,
ResourceEstimatorConfiguration.SOLVER_PROVIDER,
ResourceEstimatorConfiguration.DEFAULT_SOLVER_PROVIDER,
Solver.class);
solver.init(config, skylineStore);
} catch (Exception ex) {
LOGGER
.error("Server initialization failed due to: {}", ex.getMessage());
throw new ResourceEstimatorException(ex.getMessage(), ex);
}
gson = new GsonBuilder()
.registerTypeAdapter(Resource.class, new ResourceSerDe())
.registerTypeAdapter(RLESparseResourceAllocation.class,
new RLESparseResourceAllocationSerDe())
.enableComplexMapKeySerialization().create();
rleType = new TypeToken<RLESparseResourceAllocation>() {
}.getType();
skylineStoreType =
new TypeToken<Map<RecurrenceId, List<ResourceSkyline>>>() {
}.getType();
try {
config = new Configuration();
config.addResource(ResourceEstimatorConfiguration.CONFIG_FILE);
skylineStore = ResourceEstimatorUtil.createProviderInstance(config,
ResourceEstimatorConfiguration.SKYLINESTORE_PROVIDER,
ResourceEstimatorConfiguration.DEFAULT_SKYLINESTORE_PROVIDER,
SkylineStore.class);
logParser = ResourceEstimatorUtil.createProviderInstance(config,
ResourceEstimatorConfiguration.TRANSLATOR_PROVIDER,
ResourceEstimatorConfiguration.DEFAULT_TRANSLATOR_PROVIDER,
LogParser.class);
logParser.init(config, skylineStore);
logParserUtil.setLogParser(logParser);
solver = ResourceEstimatorUtil.createProviderInstance(config,
ResourceEstimatorConfiguration.SOLVER_PROVIDER,
ResourceEstimatorConfiguration.DEFAULT_SOLVER_PROVIDER,
Solver.class);
solver.init(config, skylineStore);
} catch (Exception ex) {
LOGGER
.error("Server initialization failed due to: {}", ex.getMessage());
throw new ResourceEstimatorException(ex.getMessage(), ex);
}
gson = new GsonBuilder()
.registerTypeAdapter(Resource.class, new ResourceSerDe())
.registerTypeAdapter(RLESparseResourceAllocation.class,
new RLESparseResourceAllocationSerDe())
.enableComplexMapKeySerialization().create();
rleType = new TypeToken<RLESparseResourceAllocation>() {
}.getType();
skylineStoreType =
new TypeToken<Map<RecurrenceId, List<ResourceSkyline>>>() {
}.getType();
}
/**
@ -192,9 +190,6 @@ public String getHistoryResourceSkyline(
LOGGER
.debug("Query the skyline store for recurrenceId: {}." + recurrenceId);
recurrenceId = new RecurrenceId("*", "*");
jobHistory = skylineStore.getHistory(recurrenceId);
return skyline;
}

5
hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/translator/impl/BaseLogParser.java
View File

@ -24,11 +24,13 @@
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.nio.charset.StandardCharsets;
import java.text.ParseException;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.commons.lang.CharSet;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.resourceestimator.common.api.RecurrenceId;
import org.apache.hadoop.resourceestimator.common.api.ResourceSkyline;
@ -101,7 +103,8 @@ public void parseLine(final String logLine,
new HashMap<>();
final Map<String, JobMetaData> jobMetas =
new HashMap<String, JobMetaData>();
final BufferedReader bf = new BufferedReader(new InputStreamReader(logs));
final BufferedReader bf = new BufferedReader(
new InputStreamReader(logs, StandardCharsets.UTF_8));
String line = null;
while ((line = bf.readLine()) != null) {
try {

11
hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/translator/impl/LogParserUtil.java
View File

@ -91,7 +91,14 @@ public final void parseLog(final String logFile)
throw new ResourceEstimatorException("The log parser is not initialized,"
+ " please try again after initializing.");
}
InputStream inputStream = new FileInputStream(logFile);
logParser.parseStream(inputStream);
InputStream inputStream = null;
try {
inputStream = new FileInputStream(logFile);
logParser.parseStream(inputStream);
} finally {
if (inputStream != null) {
inputStream.close();
}
}
}
}

2
hadoop-tools/hadoop-resourceestimator/src/test/java/org/apache/hadoop/resourceestimator/service/TestResourceEstimatorService.java
View File

@ -54,8 +54,6 @@
* Test ResourceEstimatorService.
*/
public class TestResourceEstimatorService extends JerseyTest {
private static final Logger LOGGER =
LoggerFactory.getLogger(TestResourceEstimatorService.class);
private final String parseLogCommand = "resourceestimator/translator/"
+ "src/test/resources/resourceEstimatorService.txt";
private final String getHistorySkylineCommand =