big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-18890. Remove use of okhttp in runtime code (#6057)

Browse Source 
Contributed by PJ Fanning
This commit is contained in:
PJ Fanning 2023-09-19 12:38:36 +01:00 committed by GitHub
parent 23c22b2823
commit c16484ffb2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 99 additions and 192 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
LICENSE-binary
View File

@ -241,8 +241,6 @@ com.google.guava:guava:27.0-jre
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
com.microsoft.azure:azure-storage:7.0.0
com.nimbusds:nimbus-jose-jwt:9.31
com.squareup.okhttp3:okhttp:4.10.0
com.squareup.okio:okio:3.4.0
com.zaxxer:HikariCP:4.0.3
commons-beanutils:commons-beanutils:1.9.4
commons-cli:commons-cli:1.5.0
@ -319,8 +317,8 @@ org.apache.hbase:hbase-common:1.7.1
org.apache.hbase:hbase-protocol:1.7.1
org.apache.htrace:htrace-core:3.1.0-incubating
org.apache.htrace:htrace-core4:4.1.0-incubating
org.apache.httpcomponents:httpclient:4.5.6
org.apache.httpcomponents:httpcore:4.4.10
org.apache.httpcomponents:httpclient:4.5.13
org.apache.httpcomponents:httpcore:4.4.13
org.apache.kafka:kafka-clients:2.8.2
org.apache.kerby:kerb-admin:2.0.3
org.apache.kerby:kerb-client:2.0.3
@ -357,8 +355,6 @@ org.eclipse.jetty.websocket:javax-websocket-client-impl:9.4.51.v20230217
org.eclipse.jetty.websocket:javax-websocket-server-impl:9.4.51.v20230217
org.ehcache:ehcache:3.3.1
org.ini4j:ini4j:0.5.4
org.jetbrains.kotlin:kotlin-stdlib:1.4.10
org.jetbrains.kotlin:kotlin-stdlib-common:1.4.10
org.lz4:lz4-java:1.7.1
org.objenesis:objenesis:2.6
org.xerial.snappy:snappy-java:1.1.10.1

13
NOTICE-binary
View File

@ -334,19 +334,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-----------------------------------------------------------------------
This product contains a modified portion of 'OkHttp', an open source
HTTP & SPDY client for Android and Java applications, which can be obtained
at:
* LICENSE:
* okhttp/third_party/okhttp/LICENSE (Apache License 2.0)
* HOMEPAGE:
* https://github.com/square/okhttp
* LOCATION_IN_GRPC:
* okhttp/third_party/okhttp
This product contains a modified portion of 'Netty', an open source
networking library, which can be obtained at:

12
hadoop-client-modules/hadoop-client/pom.xml
View File

@ -118,18 +118,6 @@
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-server</artifactId>
</exclusion>
<exclusion>
<groupId>org.jetbrains.kotlin</groupId>
<artifactId>kotlin-stdlib</artifactId>
</exclusion>
<exclusion>
<groupId>org.jetbrains.kotlin</groupId>
<artifactId>kotlin-stdlib-common</artifactId>
</exclusion>
<exclusion>
<groupId>com.squareup.okhttp3</groupId>
<artifactId>okhttp</artifactId>
</exclusion>
<exclusion>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-core</artifactId>

4
hadoop-cloud-storage-project/hadoop-huaweicloud/pom.xml
View File

@ -161,10 +161,6 @@
<artifactId>esdk-obs-java</artifactId>
<version>${esdk.version}</version>
<exclusions>
<exclusion>
<artifactId>okio</artifactId>
<groupId>com.squareup.okio</groupId>
</exclusion>
<exclusion>
<artifactId>log4j-core</artifactId>
<groupId>org.apache.logging.log4j</groupId>

4
hadoop-common-project/hadoop-common/pom.xml
View File

@ -394,8 +394,8 @@
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.squareup.okio</groupId>
<artifactId>okio-jvm</artifactId>
<groupId>org.jetbrains.kotlin</groupId>
<artifactId>kotlin-stdlib-jdk8</artifactId>
<scope>test</scope>
</dependency>
<dependency>

13
hadoop-hdfs-project/hadoop-hdfs-client/dev-support/findbugsExcludeFile.xml
View File

@ -94,17 +94,4 @@
<Bug pattern="EI_EXPOSE_REP" />
</Match>
<!--okhttp classes from Kotlin are not analysed for NP check. -->
<Match>
<Class name="org.apache.hadoop.hdfs.web.oauth2.ConfRefreshTokenBasedAccessTokenProvider" />
<Method name="refresh" />
<Bug pattern="NP_NULL_ON_SOME_PATH_FROM_RETURN_VALUE" />
</Match>
<Match>
<Class name="org.apache.hadoop.hdfs.web.oauth2.CredentialBasedAccessTokenProvider" />
<Method name="refresh" />
<Bug pattern="NP_NULL_ON_SOME_PATH_FROM_RETURN_VALUE" />
</Match>
</FindBugsFilter>

22
hadoop-hdfs-project/hadoop-hdfs-client/pom.xml
View File

@ -34,28 +34,6 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
</properties>
<dependencies>
<dependency>
<groupId>com.squareup.okhttp3</groupId>
<artifactId>okhttp</artifactId>
<exclusions>
<exclusion>
<groupId>com.squareup.okio</groupId>
<artifactId>okio-jvm</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>com.squareup.okio</groupId>
<artifactId>okio-jvm</artifactId>
</dependency>
<dependency>
<groupId>org.jetbrains.kotlin</groupId>
<artifactId>kotlin-stdlib</artifactId>
</dependency>
<dependency>
<groupId>org.jetbrains.kotlin</groupId>
<artifactId>kotlin-stdlib-common</artifactId>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>

72
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/ConfRefreshTokenBasedAccessTokenProvider.java
View File

@ -19,13 +19,10 @@
package org.apache.hadoop.hdfs.web.oauth2;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import okhttp3.OkHttpClient;
import okhttp3.Request;
import okhttp3.RequestBody;
import okhttp3.Response;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
@ -33,7 +30,17 @@
import org.apache.hadoop.hdfs.web.URLConnectionFactory;
import org.apache.hadoop.util.JsonSerialization;
import org.apache.hadoop.util.Timer;
import org.apache.http.HttpHeaders;
import org.apache.http.HttpStatus;
import org.apache.http.NameValuePair;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.util.EntityUtils;
import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.OAUTH_CLIENT_ID_KEY;
import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.OAUTH_REFRESH_URL_KEY;
@ -103,34 +110,37 @@ public synchronized String getAccessToken() throws IOException {
}
void refresh() throws IOException {
OkHttpClient client =
new OkHttpClient.Builder().connectTimeout(URLConnectionFactory.DEFAULT_SOCKET_TIMEOUT,
TimeUnit.MILLISECONDS)
.readTimeout(URLConnectionFactory.DEFAULT_SOCKET_TIMEOUT, TimeUnit.MILLISECONDS)
.build();
final List<NameValuePair> pairs = new ArrayList<>();
pairs.add(new BasicNameValuePair(GRANT_TYPE, REFRESH_TOKEN));
pairs.add(new BasicNameValuePair(REFRESH_TOKEN, refreshToken));
pairs.add(new BasicNameValuePair(CLIENT_ID, clientId));
final RequestConfig config = RequestConfig.custom()
.setConnectTimeout(URLConnectionFactory.DEFAULT_SOCKET_TIMEOUT)
.setConnectionRequestTimeout(URLConnectionFactory.DEFAULT_SOCKET_TIMEOUT)
.setSocketTimeout(URLConnectionFactory.DEFAULT_SOCKET_TIMEOUT)
.build();
try (CloseableHttpClient client =
HttpClientBuilder.create().setDefaultRequestConfig(config).build()) {
final HttpPost httpPost = new HttpPost(refreshURL);
httpPost.setEntity(new UrlEncodedFormEntity(pairs, StandardCharsets.UTF_8));
httpPost.setHeader(HttpHeaders.CONTENT_TYPE, URLENCODED);
try (CloseableHttpResponse response = client.execute(httpPost)) {
final int statusCode = response.getStatusLine().getStatusCode();
if (statusCode != HttpStatus.SC_OK) {
throw new IllegalArgumentException(
"Received invalid http response: " + statusCode + ", text = " +
EntityUtils.toString(response.getEntity()));
}
Map<?, ?> responseBody = JsonSerialization.mapReader().readValue(
EntityUtils.toString(response.getEntity()));
String bodyString =
Utils.postBody(GRANT_TYPE, REFRESH_TOKEN, REFRESH_TOKEN, refreshToken, CLIENT_ID, clientId);
String newExpiresIn = responseBody.get(EXPIRES_IN).toString();
accessTokenTimer.setExpiresIn(newExpiresIn);
RequestBody body = RequestBody.create(bodyString, URLENCODED);
Request request = new Request.Builder().url(refreshURL).post(body).build();
try (Response response = client.newCall(request).execute()) {
if (!response.isSuccessful()) {
throw new IOException("Unexpected code " + response);
accessToken = responseBody.get(ACCESS_TOKEN).toString();
}
if (response.code() != HttpStatus.SC_OK) {
throw new IllegalArgumentException(
"Received invalid http response: " + response.code() + ", text = "
+ response.toString());
}
Map<?, ?> responseBody = JsonSerialization.mapReader().readValue(response.body().string());
String newExpiresIn = responseBody.get(EXPIRES_IN).toString();
accessTokenTimer.setExpiresIn(newExpiresIn);
accessToken = responseBody.get(ACCESS_TOKEN).toString();
} catch (RuntimeException e) {
throw new IOException("Exception while refreshing access token", e);
} catch (Exception e) {
throw new IOException("Exception while refreshing access token", e);
}

78
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/CredentialBasedAccessTokenProvider.java
View File

@ -19,13 +19,10 @@
package org.apache.hadoop.hdfs.web.oauth2;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import okhttp3.OkHttpClient;
import okhttp3.Request;
import okhttp3.RequestBody;
import okhttp3.Response;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
@ -33,7 +30,17 @@
import org.apache.hadoop.hdfs.web.URLConnectionFactory;
import org.apache.hadoop.util.JsonSerialization;
import org.apache.hadoop.util.Timer;
import org.apache.http.HttpHeaders;
import org.apache.http.HttpStatus;
import org.apache.http.NameValuePair;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.util.EntityUtils;
import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.OAUTH_CLIENT_ID_KEY;
import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.OAUTH_REFRESH_URL_KEY;
@ -97,38 +104,37 @@ public synchronized String getAccessToken() throws IOException {
}
void refresh() throws IOException {
OkHttpClient client = new OkHttpClient.Builder()
.connectTimeout(URLConnectionFactory.DEFAULT_SOCKET_TIMEOUT, TimeUnit.MILLISECONDS)
.readTimeout(URLConnectionFactory.DEFAULT_SOCKET_TIMEOUT, TimeUnit.MILLISECONDS)
.build();
String bodyString = Utils.postBody(CLIENT_SECRET, getCredential(),
GRANT_TYPE, CLIENT_CREDENTIALS,
CLIENT_ID, clientId);
RequestBody body = RequestBody.create(bodyString, URLENCODED);
Request request = new Request.Builder()
.url(refreshURL)
.post(body)
final List<NameValuePair> pairs = new ArrayList<>();
pairs.add(new BasicNameValuePair(CLIENT_SECRET, getCredential()));
pairs.add(new BasicNameValuePair(GRANT_TYPE, CLIENT_CREDENTIALS));
pairs.add(new BasicNameValuePair(CLIENT_ID, clientId));
final RequestConfig config = RequestConfig.custom()
.setConnectTimeout(URLConnectionFactory.DEFAULT_SOCKET_TIMEOUT)
.setConnectionRequestTimeout(URLConnectionFactory.DEFAULT_SOCKET_TIMEOUT)
.setSocketTimeout(URLConnectionFactory.DEFAULT_SOCKET_TIMEOUT)
.build();
try (Response response = client.newCall(request).execute()) {
if (!response.isSuccessful()) {
throw new IOException("Unexpected code " + response);
try (CloseableHttpClient client =
HttpClientBuilder.create().setDefaultRequestConfig(config).build()) {
final HttpPost httpPost = new HttpPost(refreshURL);
httpPost.setEntity(new UrlEncodedFormEntity(pairs, StandardCharsets.UTF_8));
httpPost.setHeader(HttpHeaders.CONTENT_TYPE, URLENCODED);
try (CloseableHttpResponse response = client.execute(httpPost)) {
final int statusCode = response.getStatusLine().getStatusCode();
if (statusCode != HttpStatus.SC_OK) {
throw new IllegalArgumentException(
"Received invalid http response: " + statusCode + ", text = " +
EntityUtils.toString(response.getEntity()));
}
Map<?, ?> responseBody = JsonSerialization.mapReader().readValue(
EntityUtils.toString(response.getEntity()));
String newExpiresIn = responseBody.get(EXPIRES_IN).toString();
timer.setExpiresIn(newExpiresIn);
accessToken = responseBody.get(ACCESS_TOKEN).toString();
}
if (response.code() != HttpStatus.SC_OK) {
throw new IllegalArgumentException("Received invalid http response: "
+ response.code() + ", text = " + response.toString());
}
Map<?, ?> responseBody = JsonSerialization.mapReader().readValue(
response.body().string());
String newExpiresIn = responseBody.get(EXPIRES_IN).toString();
timer.setExpiresIn(newExpiresIn);
accessToken = responseBody.get(ACCESS_TOKEN).toString();
} catch (RuntimeException e) {
throw new IOException("Unable to obtain access token from credential", e);
} catch (Exception e) {
throw new IOException("Unable to obtain access token from credential", e);
}

5
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/OAuth2Constants.java
View File

@ -18,7 +18,6 @@
*/
package org.apache.hadoop.hdfs.web.oauth2;
import okhttp3.MediaType;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
@ -30,8 +29,8 @@
public final class OAuth2Constants {
private OAuth2Constants() { /** Private constructor. **/ }
public static final MediaType URLENCODED
= MediaType.parse("application/x-www-form-urlencoded; charset=utf-8");
public static final String URLENCODED
= "application/x-www-form-urlencoded; charset=utf-8";
/* Constants for OAuth protocol */
public static final String ACCESS_TOKEN = "access_token";

4
hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml
View File

@ -205,8 +205,8 @@
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.squareup.okhttp3</groupId>
<artifactId>okhttp</artifactId>
<groupId>org.jetbrains.kotlin</groupId>
<artifactId>kotlin-stdlib-jdk8</artifactId>
<scope>test</scope>
</dependency>
</dependencies>

56
hadoop-project/pom.xml
View File

@ -131,10 +131,8 @@
<hikari.version>4.0.3</hikari.version>
<derby.version>10.14.2.0</derby.version>
<mssql.version>6.2.1.jre7</mssql.version>
<okhttp3.version>4.10.0</okhttp3.version>
<okio.version>3.4.0</okio.version>
<kotlin-stdlib.verion>1.6.20</kotlin-stdlib.verion>
<kotlin-stdlib-common.version>1.6.20</kotlin-stdlib-common.version>
<okhttp3.version>4.11.0</okhttp3.version>
<kotlin-stdlib.version>1.6.20</kotlin-stdlib.version>
<jdom2.version>2.0.6.1</jdom2.version>
<jna.version>5.2.0</jna.version>
<gson.version>2.9.0</gson.version>
@ -220,62 +218,24 @@
<dependencyManagement>
<dependencies>
<dependency>
<groupId>com.squareup.okhttp3</groupId>
<artifactId>okhttp</artifactId>
<version>${okhttp3.version}</version>
<exclusions>
<exclusion>
<groupId>org.jetbrains.kotlin</groupId>
<artifactId>kotlin-stdlib</artifactId>
</exclusion>
<exclusion>
<groupId>org.jetbrains.kotlin</groupId>
<artifactId>kotlin-stdlib-common</artifactId>
</exclusion>
<exclusion>
<groupId>com.squareup.okio</groupId>
<artifactId>okio-jvm</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>com.squareup.okio</groupId>
<artifactId>okio-jvm</artifactId>
<version>${okio.version}</version>
</dependency>
<dependency>
<groupId>org.jetbrains.kotlin</groupId>
<artifactId>kotlin-stdlib</artifactId>
<version>${kotlin-stdlib.verion}</version>
<exclusions>
<exclusion>
<groupId>org.jetbrains</groupId>
<artifactId>annotations</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.jetbrains.kotlin</groupId>
<artifactId>kotlin-stdlib-common</artifactId>
<version>${kotlin-stdlib-common.version}</version>
</dependency>
<dependency>
<groupId>com.squareup.okhttp3</groupId>
<artifactId>mockwebserver</artifactId>
<version>${okhttp3.version}</version>
<scope>test</scope>
<exclusions>
<exclusion>
<groupId>com.squareup.okio</groupId>
<artifactId>okio-jvm</artifactId>
</exclusion>
<exclusion>
<groupId>org.jetbrains.kotlin</groupId>
<artifactId>kotlin-stdlib-jdk8</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.jetbrains.kotlin</groupId>
<artifactId>kotlin-stdlib-jdk8</artifactId>
<version>${kotlin-stdlib.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>jdiff</groupId>
<artifactId>jdiff</artifactId>