big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-10465. Fix use of generics within SortedMapWritable. Contributed by Bertrand Dechoux.

Browse Source
This commit is contained in:
Haohui Mai 2015-11-22 18:10:08 -08:00
parent aac260faa1
commit dc46c46b91
5 changed files with 53 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-common-project/hadoop-common/CHANGES.txt
View File

@ -978,6 +978,9 @@ Release 2.8.0 - UNRELEASED
HADOOP-10555. Add offset support to MurmurHash.
(Sergey Shelukhin via wheat9)
HADOOP-10465. Fix use of generics within SortedMapWritable.
(Bertrand Dechoux via wheat9)
OPTIMIZATIONS
HADOOP-11785. Reduce the number of listStatus operation in distcp

47
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SortedMapWritable.java
View File

@ -36,15 +36,15 @@
*/
@InterfaceAudience.Public
@InterfaceStability.Stable
public class SortedMapWritable extends AbstractMapWritable
implements SortedMap<WritableComparable, Writable> {
public class SortedMapWritable<K extends WritableComparable<? super K>> extends AbstractMapWritable
implements SortedMap<K, Writable> {
private SortedMap<WritableComparable, Writable> instance;
private SortedMap<K, Writable> instance;
/** default constructor. */
public SortedMapWritable() {
super();
this.instance = new TreeMap<WritableComparable, Writable>();
this.instance = new TreeMap<K, Writable>();
}
/**
@ -52,45 +52,39 @@ public SortedMapWritable() {
*
* @param other the map to copy from
*/
public SortedMapWritable(SortedMapWritable other) {
public SortedMapWritable(SortedMapWritable<K> other) {
this();
copy(other);
}
@Override
public Comparator<? super WritableComparable> comparator() {
public Comparator<? super K> comparator() {
// Returning null means we use the natural ordering of the keys
return null;
}
@Override
public WritableComparable firstKey() {
public K firstKey() {
return instance.firstKey();
}
@Override
public SortedMap<WritableComparable, Writable>
headMap(WritableComparable toKey) {
public SortedMap<K, Writable> headMap(K toKey) {
return instance.headMap(toKey);
}
@Override
public WritableComparable lastKey() {
public K lastKey() {
return instance.lastKey();
}
@Override
public SortedMap<WritableComparable, Writable>
subMap(WritableComparable fromKey, WritableComparable toKey) {
public SortedMap<K, Writable> subMap(K fromKey, K toKey) {
return instance.subMap(fromKey, toKey);
}
@Override
public SortedMap<WritableComparable, Writable>
tailMap(WritableComparable fromKey) {
public SortedMap<K, Writable> tailMap(K fromKey) {
return instance.tailMap(fromKey);
}
@ -110,7 +104,7 @@ public boolean containsValue(Object value) {
}
@Override
public Set<java.util.Map.Entry<WritableComparable, Writable>> entrySet() {
public Set<Map.Entry<K, Writable>> entrySet() {
return instance.entrySet();
}
@ -125,22 +119,21 @@ public boolean isEmpty() {
}
@Override
public Set<WritableComparable> keySet() {
public Set<K> keySet() {
return instance.keySet();
}
@Override
public Writable put(WritableComparable key, Writable value) {
public Writable put(K key, Writable value) {
addToMap(key.getClass());
addToMap(value.getClass());
return instance.put(key, value);
}
@Override
public void putAll(Map<? extends WritableComparable, ? extends Writable> t) {
for (Map.Entry<? extends WritableComparable, ? extends Writable> e:
public void putAll(Map<? extends K, ? extends Writable> t) {
for (Map.Entry<? extends K, ? extends Writable> e:
t.entrySet()) {
put(e.getKey(), e.getValue());
}
}
@ -172,8 +165,8 @@ public void readFields(DataInput in) throws IOException {
// Then read each key/value pair
for (int i = 0; i < entries; i++) {
WritableComparable key =
(WritableComparable) ReflectionUtils.newInstance(getClass(
K key =
(K) ReflectionUtils.newInstance(getClass(
in.readByte()), getConf());
key.readFields(in);
@ -196,7 +189,7 @@ public void write(DataOutput out) throws IOException {
// Then write out each key/value pair
for (Map.Entry<WritableComparable, Writable> e: instance.entrySet()) {
for (Map.Entry<K, Writable> e: instance.entrySet()) {
out.writeByte(getId(e.getKey().getClass()));
e.getKey().write(out);
out.writeByte(getId(e.getValue().getClass()));
@ -211,7 +204,7 @@ public boolean equals(Object obj) {
}
if (obj instanceof SortedMapWritable) {
Map map = (Map) obj;
Map<?,?> map = (Map<?,?>) obj;
if (size() != map.size()) {
return false;
}

37
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestSortedMapWritable.java
View File

@ -45,7 +45,7 @@ public void testSortedMapWritable() {
new BytesWritable("value3".getBytes())
};
SortedMapWritable inMap = new SortedMapWritable();
SortedMapWritable<Text> inMap = new SortedMapWritable<Text>();
for (int i = 0; i < keys.length; i++) {
inMap.put(keys[i], values[i]);
}
@ -53,13 +53,14 @@ public void testSortedMapWritable() {
assertEquals(0, inMap.firstKey().compareTo(keys[0]));
assertEquals(0, inMap.lastKey().compareTo(keys[2]));
SortedMapWritable outMap = new SortedMapWritable(inMap);
SortedMapWritable<Text> outMap = new SortedMapWritable<Text>(inMap);
assertEquals(inMap.size(), outMap.size());
for (Map.Entry<WritableComparable, Writable> e: inMap.entrySet()) {
for (Map.Entry<Text, Writable> e: inMap.entrySet()) {
assertTrue(outMap.containsKey(e.getKey()));
assertEquals(0, ((WritableComparable) outMap.get(e.getKey())).compareTo(
e.getValue()));
WritableComparable<WritableComparable<?>> aValue = (WritableComparable<WritableComparable<?>>) outMap.get(e.getKey());
WritableComparable<WritableComparable<?>> bValue = (WritableComparable<WritableComparable<?>>) e.getValue();
assertEquals(0, aValue.compareTo(bValue));
}
// Now for something a little harder...
@ -69,24 +70,24 @@ public void testSortedMapWritable() {
new Text("map2")
};
SortedMapWritable mapOfMaps = new SortedMapWritable();
SortedMapWritable<Text> mapOfMaps = new SortedMapWritable<Text>();
mapOfMaps.put(maps[0], inMap);
mapOfMaps.put(maps[1], outMap);
SortedMapWritable copyOfMapOfMaps = new SortedMapWritable(mapOfMaps);
SortedMapWritable<Text> copyOfMapOfMaps = new SortedMapWritable<Text>(mapOfMaps);
for (int i = 0; i < maps.length; i++) {
assertTrue(copyOfMapOfMaps.containsKey(maps[i]));
SortedMapWritable a = (SortedMapWritable) mapOfMaps.get(maps[i]);
SortedMapWritable b = (SortedMapWritable) copyOfMapOfMaps.get(maps[i]);
SortedMapWritable<Text> a = (SortedMapWritable<Text>) mapOfMaps.get(maps[i]);
SortedMapWritable<Text> b = (SortedMapWritable<Text>) copyOfMapOfMaps.get(maps[i]);
assertEquals(a.size(), b.size());
for (Writable key: a.keySet()) {
assertTrue(b.containsKey(key));
// This will work because we know what we put into each set
WritableComparable aValue = (WritableComparable) a.get(key);
WritableComparable bValue = (WritableComparable) b.get(key);
WritableComparable<WritableComparable<?>> aValue = (WritableComparable<WritableComparable<?>>) a.get(key);
WritableComparable<WritableComparable<?>> bValue = (WritableComparable<WritableComparable<?>>) b.get(key);
assertEquals(0, aValue.compareTo(bValue));
}
}
@ -98,11 +99,11 @@ public void testSortedMapWritable() {
@Test
@SuppressWarnings("deprecation")
public void testForeignClass() {
SortedMapWritable inMap = new SortedMapWritable();
SortedMapWritable<Text> inMap = new SortedMapWritable<Text>();
inMap.put(new Text("key"), new UTF8("value"));
inMap.put(new Text("key2"), new UTF8("value2"));
SortedMapWritable outMap = new SortedMapWritable(inMap);
SortedMapWritable copyOfCopy = new SortedMapWritable(outMap);
SortedMapWritable<Text> outMap = new SortedMapWritable<Text>(inMap);
SortedMapWritable<Text> copyOfCopy = new SortedMapWritable<Text>(outMap);
assertEquals(1, copyOfCopy.getNewClasses());
}
@ -112,8 +113,8 @@ public void testForeignClass() {
@Test
public void testEqualsAndHashCode() {
String failureReason;
SortedMapWritable mapA = new SortedMapWritable();
SortedMapWritable mapB = new SortedMapWritable();
SortedMapWritable<Text> mapA = new SortedMapWritable<Text>();
SortedMapWritable<Text> mapB = new SortedMapWritable<Text>();
// Sanity checks
failureReason = "SortedMapWritable couldn't be initialized. Got null reference";
@ -167,8 +168,8 @@ public void testEqualsAndHashCode() {
@Test(timeout = 1000)
public void testPutAll() {
SortedMapWritable map1 = new SortedMapWritable();
SortedMapWritable map2 = new SortedMapWritable();
SortedMapWritable<Text> map1 = new SortedMapWritable<Text>();
SortedMapWritable<Text> map2 = new SortedMapWritable<Text>();
map1.put(new Text("key"), new Text("value"));
map2.putAll(map1);

13
hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/typedbytes/TypedBytesWritableInput.java
View File

@ -330,22 +330,25 @@ public MapWritable readMap() throws IOException {
return readMap(null);
}
public SortedMapWritable readSortedMap(SortedMapWritable mw)
public <K extends WritableComparable<? super K>>
SortedMapWritable<K> readSortedMap(SortedMapWritable<K> mw)
throws IOException {
if (mw == null) {
mw = new SortedMapWritable();
mw = new SortedMapWritable<K>();
}
int length = in.readMapHeader();
for (int i = 0; i < length; i++) {
WritableComparable key = (WritableComparable) read();
@SuppressWarnings("unchecked")
K key = (K) read();
Writable value = read();
mw.put(key, value);
}
return mw;
}
public SortedMapWritable readSortedMap() throws IOException {
return readSortedMap(null);
public <K extends WritableComparable<? super K>> SortedMapWritable<K>
readSortedMap() throws IOException {
return readSortedMap((SortedMapWritable<K>)null);
}
public Writable readWritable(Writable writable) throws IOException {

6
hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/typedbytes/TypedBytesWritableOutput.java
View File

@ -131,7 +131,7 @@ public void write(Writable w) throws IOException {
} else if (w instanceof MapWritable) {
writeMap((MapWritable) w);
} else if (w instanceof SortedMapWritable) {
writeSortedMap((SortedMapWritable) w);
writeSortedMap((SortedMapWritable<?>) w);
} else if (w instanceof Record) {
writeRecord((Record) w);
} else {
@ -200,9 +200,9 @@ public void writeMap(MapWritable mw) throws IOException {
}
}
public void writeSortedMap(SortedMapWritable smw) throws IOException {
public void writeSortedMap(SortedMapWritable<?> smw) throws IOException {
out.writeMapHeader(smw.size());
for (Map.Entry<WritableComparable, Writable> entry : smw.entrySet()) {
for (Map.Entry<? extends WritableComparable<?>, Writable> entry : smw.entrySet()) {
write(entry.getKey());
write(entry.getValue());
}