big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-17870. Http Filesystem to qualify relative paths. (#3338)

Browse Source 
Contributed by Yellowflash
This commit is contained in:
Yellow Flash 2021-08-31 18:25:52 +05:30 committed by GitHub
parent 164608b546
commit 4ea60b5733
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 36 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/http/AbstractHttpFileSystem.java
View File

@ -60,7 +60,8 @@ public URI getUri() {
@Override
public FSDataInputStream open(Path path, int bufferSize) throws IOException {
URLConnection conn = path.toUri().toURL().openConnection();
URI pathUri = makeQualified(path).toUri();
URLConnection conn = pathUri.toURL().openConnection();
InputStream in = conn.getInputStream();
return new FSDataInputStream(new HttpDataInputStream(in));
}
@ -111,7 +112,7 @@ public boolean mkdirs(Path path, FsPermission fsPermission)
@Override
public FileStatus getFileStatus(Path path) throws IOException {
return new FileStatus(-1, false, 1, DEFAULT_BLOCK_SIZE, 0, path);
return new FileStatus(-1, false, 1, DEFAULT_BLOCK_SIZE, 0, makeQualified(path));
}
/**

44
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/http/TestHttpFileSystem.java
View File

@ -26,6 +26,7 @@
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;
import org.junit.Test;
import org.junit.jupiter.api.BeforeEach;
import java.io.IOException;
import java.io.InputStream;
@ -33,6 +34,7 @@
import java.net.URISyntaxException;
import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.util.stream.IntStream;
import static org.junit.Assert.assertEquals;
@ -40,28 +42,48 @@
* Testing HttpFileSystem.
*/
public class TestHttpFileSystem {
private final Configuration conf = new Configuration(false);
@BeforeEach
public void setUp() {
conf.set("fs.http.impl", HttpFileSystem.class.getCanonicalName());
}
@Test
public void testHttpFileSystem() throws IOException, URISyntaxException,
InterruptedException {
Configuration conf = new Configuration(false);
conf.set("fs.http.impl", HttpFileSystem.class.getCanonicalName());
final String data = "foo";
try (MockWebServer server = new MockWebServer()) {
server.enqueue(new MockResponse().setBody(data));
IntStream.rangeClosed(1, 3).forEach(i -> server.enqueue(new MockResponse().setBody(data)));
server.start();
URI uri = URI.create(String.format("http://%s:%d", server.getHostName(),
server.getPort()));
FileSystem fs = FileSystem.get(uri, conf);
try (InputStream is = fs.open(
new Path(new URL(uri.toURL(), "/foo").toURI()),
4096)) {
byte[] buf = new byte[data.length()];
IOUtils.readFully(is, buf, 0, buf.length);
assertEquals(data, new String(buf, StandardCharsets.UTF_8));
}
assertSameData(fs, new Path(new URL(uri.toURL(), "/foo").toURI()), data);
assertSameData(fs, new Path("/foo"), data);
assertSameData(fs, new Path("foo"), data);
RecordedRequest req = server.takeRequest();
assertEquals("/foo", req.getPath());
}
}
@Test
public void testHttpFileStatus() throws IOException, URISyntaxException, InterruptedException {
URI uri = new URI("http://www.example.com");
FileSystem fs = FileSystem.get(uri, conf);
URI expectedUri = uri.resolve("/foo");
assertEquals(fs.getFileStatus(new Path(new Path(uri), "/foo")).getPath().toUri(), expectedUri);
assertEquals(fs.getFileStatus(new Path("/foo")).getPath().toUri(), expectedUri);
assertEquals(fs.getFileStatus(new Path("foo")).getPath().toUri(), expectedUri);
}
private void assertSameData(FileSystem fs, Path path, String data) throws IOException {
try (InputStream is = fs.open(
path,
4096)) {
byte[] buf = new byte[data.length()];
IOUtils.readFully(is, buf, 0, buf.length);
assertEquals(data, new String(buf, StandardCharsets.UTF_8));
}
}
}