big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

MAPREDUCE-6237. Multiple mappers with DBInputFormat don't work because of reusing conections. Contributed by Kannan Rajah.

Browse Source
This commit is contained in:
Tsuyoshi Ozawa 2015-02-10 03:52:42 +09:00
parent 7e42088abf
commit 241336ca2b
4 changed files with 37 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
hadoop-mapreduce-project/CHANGES.txt
View File

@ -346,6 +346,21 @@ Release 2.7.0 - UNRELEASED
MAPREDUCE-6233. org.apache.hadoop.mapreduce.TestLargeSort.testLargeSort
failed in trunk (zxu via rkanter)
Release 2.6.1 - UNRELEASED
INCOMPATIBLE CHANGES
NEW FEATURES
IMPROVEMENTS
OPTIMIZATIONS
BUG FIXES
MAPREDUCE-6237. Multiple mappers with DBInputFormat don't work because of
reusing conections. (Kannan Rajah via ozawa)
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES

31
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/db/DBInputFormat.java
View File

@ -159,7 +159,7 @@ public void setConf(Configuration conf) {
dbConf = new DBConfiguration(conf);
try {
getConnection();
this.connection = createConnection();
DatabaseMetaData dbMeta = connection.getMetaData();
this.dbProductName = dbMeta.getDatabaseProductName().toUpperCase();
@ -182,18 +182,25 @@ public DBConfiguration getDBConf() {
}
public Connection getConnection() {
// TODO Remove this code that handles backward compatibility.
if (this.connection == null) {
this.connection = createConnection();
}
return this.connection;
}
public Connection createConnection() {
try {
if (null == this.connection) {
// The connection was closed; reinstantiate it.
this.connection = dbConf.getConnection();
this.connection.setAutoCommit(false);
this.connection.setTransactionIsolation(
Connection.TRANSACTION_SERIALIZABLE);
}
Connection newConnection = dbConf.getConnection();
newConnection.setAutoCommit(false);
newConnection.setTransactionIsolation(
Connection.TRANSACTION_SERIALIZABLE);
return newConnection;
} catch (Exception e) {
throw new RuntimeException(e);
}
return connection;
}
public String getDBProductName() {
@ -210,17 +217,17 @@ protected RecordReader<LongWritable, T> createDBRecordReader(DBInputSplit split,
if (dbProductName.startsWith("ORACLE")) {
// use Oracle-specific db reader.
return new OracleDBRecordReader<T>(split, inputClass,
conf, getConnection(), getDBConf(), conditions, fieldNames,
conf, createConnection(), getDBConf(), conditions, fieldNames,
tableName);
} else if (dbProductName.startsWith("MYSQL")) {
// use MySQL-specific db reader.
return new MySQLDBRecordReader<T>(split, inputClass,
conf, getConnection(), getDBConf(), conditions, fieldNames,
conf, createConnection(), getDBConf(), conditions, fieldNames,
tableName);
} else {
// Generic reader.
return new DBRecordReader<T>(split, inputClass,
conf, getConnection(), getDBConf(), conditions, fieldNames,
conf, createConnection(), getDBConf(), conditions, fieldNames,
tableName);
}
} catch (SQLException ex) {

5
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/db/DataDrivenDBInputFormat.java
View File

@ -178,7 +178,6 @@ public List<InputSplit> getSplits(JobContext job) throws IOException {
ResultSet results = null;
Statement statement = null;
Connection connection = getConnection();
try {
statement = connection.createStatement();
@ -289,12 +288,12 @@ protected RecordReader<LongWritable, T> createDBRecordReader(DBInputSplit split,
if (dbProductName.startsWith("MYSQL")) {
// use MySQL-specific db reader.
return new MySQLDataDrivenDBRecordReader<T>(split, inputClass,
conf, getConnection(), dbConf, dbConf.getInputConditions(),
conf, createConnection(), dbConf, dbConf.getInputConditions(),
dbConf.getInputFieldNames(), dbConf.getInputTableName());
} else {
// Generic reader.
return new DataDrivenDBRecordReader<T>(split, inputClass,
conf, getConnection(), dbConf, dbConf.getInputConditions(),
conf, createConnection(), dbConf, dbConf.getInputConditions(),
dbConf.getInputFieldNames(), dbConf.getInputTableName(),
dbProductName);
}

2
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/db/OracleDataDrivenDBInputFormat.java
View File

@ -84,7 +84,7 @@ protected RecordReader<LongWritable, T> createDBRecordReader(DBInputSplit split,
try {
// Use Oracle-specific db reader
return new OracleDataDrivenDBRecordReader<T>(split, inputClass,
conf, getConnection(), dbConf, dbConf.getInputConditions(),
conf, createConnection(), dbConf, dbConf.getInputConditions(),
dbConf.getInputFieldNames(), dbConf.getInputTableName());
} catch (SQLException ex) {
throw new IOException(ex.getMessage());