big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-6522. Fix decoding of codepoint zero in UTF8.

Browse Source 
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@908661 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Doug Cutting 2010-02-10 21:58:11 +00:00
parent cfbdd1aff4
commit 8061821ca2
3 changed files with 17 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
CHANGES.txt
View File

@ -184,6 +184,8 @@ Trunk (unreleased changes)
HADOOP-6540. Contrib unit tests have invalid XML for core-site, etc.
(Aaron Kimball via tomwhite)
HADOOP-6522. Fix decoding of codepoint zero in UTF8. (cutting)
Release 0.21.0 - Unreleased
INCOMPATIBLE CHANGES

4
src/java/org/apache/hadoop/io/UTF8.java
View File

@ -253,7 +253,7 @@ private static int utf8Length(String string) {
int utf8Length = 0;
for (int i = 0; i < stringLength; i++) {
int c = string.charAt(i);
if ((c >= 0x0001) && (c <= 0x007F)) {
if (c <= 0x007F) {
utf8Length++;
} else if (c > 0x07FF) {
utf8Length += 3;
@ -270,7 +270,7 @@ private static void writeChars(DataOutput out,
final int end = start + length;
for (int i = start; i < end; i++) {
int code = s.charAt(i);
if (code >= 0x01 && code <= 0x7F) {
if (code <= 0x7F) {
out.writeByte((byte)code);
} else if (code <= 0x07FF) {
out.writeByte((byte)(0xC0 | ((code >> 6) & 0x1F)));

16
src/test/core/org/apache/hadoop/io/TestUTF8.java
View File

@ -22,6 +22,7 @@
import java.util.Random;
/** Unit tests for UTF8. */
@SuppressWarnings("deprecation")
public class TestUTF8 extends TestCase {
public TestUTF8(String name) { super(name); }
@ -37,13 +38,13 @@ public static String getTestString() throws Exception {
}
public void testWritable() throws Exception {
for (int i = 0; i < 10; i++) {
for (int i = 0; i < 10000; i++) {
TestWritable.testWritable(new UTF8(getTestString()));
}
}
public void testGetBytes() throws Exception {
for (int i = 0; i < 10; i++) {
for (int i = 0; i < 10000; i++) {
// generate a random string
String before = getTestString();
@ -57,7 +58,7 @@ public void testIO() throws Exception {
DataOutputBuffer out = new DataOutputBuffer();
DataInputBuffer in = new DataInputBuffer();
for (int i = 0; i < 10; i++) {
for (int i = 0; i < 10000; i++) {
// generate a random string
String before = getTestString();
@ -82,5 +83,14 @@ public void testIO() throws Exception {
}
}
public void testNullEncoding() throws Exception {
String s = new String(new char[] { 0 });
DataOutputBuffer dob = new DataOutputBuffer();
new UTF8(s).write(dob);
assertEquals(s, new String(dob.getData(), 2, dob.getLength()-2, "UTF-8"));
}
}