big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-12582. Using BytesWritable's getLength() and getBytes() instead of get() and getSize(). Contributed by Akira AJISAKA.

Browse Source
This commit is contained in:
Tsuyoshi Ozawa 2015-11-19 17:26:23 +09:00
parent 23a130abd7
commit bd166f0eed
5 changed files with 24 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-common-project/hadoop-common/CHANGES.txt
View File

@ -962,6 +962,9 @@ Release 2.8.0 - UNRELEASED
HADOOP-12564. Upgrade JUnit3 TestCase to JUnit 4 in
org.apache.hadoop.io package. (Dustin Cote via ozawa)
HADOOP-12582. Using BytesWritable's getLength() and getBytes() instead
of get() and getSize(). (Akira AJISAKA via ozawa)
OPTIMIZATIONS
HADOOP-11785. Reduce the number of listStatus operation in distcp

13
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/KVGenerator.java
View File

@ -58,17 +58,16 @@ private void fillKey(BytesWritable o) {
while (n < len) {
byte[] word = dict[random.nextInt(dict.length)];
int l = Math.min(word.length, len - n);
System.arraycopy(word, 0, o.get(), n, l);
System.arraycopy(word, 0, o.getBytes(), n, l);
n += l;
}
if (sorted
&& WritableComparator.compareBytes(lastKey.get(), MIN_KEY_LEN, lastKey
.getSize()
- MIN_KEY_LEN, o.get(), MIN_KEY_LEN, o.getSize() - MIN_KEY_LEN) > 0) {
if (sorted && WritableComparator.compareBytes(
lastKey.getBytes(), MIN_KEY_LEN, lastKey.getLength() - MIN_KEY_LEN,
o.getBytes(), MIN_KEY_LEN, o.getLength() - MIN_KEY_LEN) > 0) {
incrementPrefix();
}
System.arraycopy(prefix, 0, o.get(), 0, MIN_KEY_LEN);
System.arraycopy(prefix, 0, o.getBytes(), 0, MIN_KEY_LEN);
lastKey.set(o);
}
@ -79,7 +78,7 @@ private void fillValue(BytesWritable o) {
while (n < len) {
byte[] word = dict[random.nextInt(dict.length)];
int l = Math.min(word.length, len - n);
System.arraycopy(word, 0, o.get(), n, l);
System.arraycopy(word, 0, o.getBytes(), n, l);
n += l;
}
}

4
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/KeySampler.java
View File

@ -45,9 +45,9 @@ private int keyPrefixToInt(RawComparable key) throws IOException {
public void next(BytesWritable key) {
key.setSize(Math.max(MIN_KEY_LEN, keyLenRNG.nextInt()));
random.nextBytes(key.get());
random.nextBytes(key.getBytes());
int n = random.nextInt(max - min) + min;
byte[] b = key.get();
byte[] b = key.getBytes();
b[0] = (byte) (n >> 24);
b[1] = (byte) (n >> 16);
b[2] = (byte) (n >> 8);

14
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/TestTFileSeek.java
View File

@ -119,10 +119,10 @@ private void createTFile() throws IOException {
}
}
kvGen.next(key, val, false);
writer.append(key.get(), 0, key.getSize(), val.get(), 0, val
.getSize());
totalBytes += key.getSize();
totalBytes += val.getSize();
writer.append(key.getBytes(), 0, key.getLength(), val.getBytes(), 0,
val.getLength());
totalBytes += key.getLength();
totalBytes += val.getLength();
}
timer.stop();
}
@ -160,11 +160,11 @@ public void seekTFile() throws IOException {
timer.start();
for (int i = 0; i < options.seekCount; ++i) {
kSampler.next(key);
scanner.lowerBound(key.get(), 0, key.getSize());
scanner.lowerBound(key.getBytes(), 0, key.getLength());
if (!scanner.atEnd()) {
scanner.entry().get(key, val);
totalBytes += key.getSize();
totalBytes += val.getSize();
totalBytes += key.getLength();
totalBytes += val.getLength();
}
else {
++miss;

12
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/TestTFileSeqFileComparison.java
View File

@ -153,8 +153,8 @@ public TFileAppendable(FileSystem fs, Path path, String compress,
@Override
public void append(BytesWritable key, BytesWritable value)
throws IOException {
writer.append(key.get(), 0, key.getSize(), value.get(), 0, value
.getSize());
writer.append(key.getBytes(), 0, key.getLength(), value.getBytes(), 0,
value.getLength());
}
@Override
@ -303,22 +303,22 @@ public SeqFileReadable(FileSystem fs, Path path, int osBufferSize)
@Override
public byte[] getKey() {
return key.get();
return key.getBytes();
}
@Override
public int getKeyLength() {
return key.getSize();
return key.getLength();
}
@Override
public byte[] getValue() {
return value.get();
return value.getBytes();
}
@Override
public int getValueLength() {
return value.getSize();
return value.getLength();
}
@Override