Skip to content

Commit 8464be0

Browse files
mohsenrezaeithemohsenrezaeithe
committed
implemented Hadoop 3.4 module + migrated Azure to it
1 parent e71204f commit 8464be0

File tree

7 files changed

+466
-26
lines changed

7 files changed

+466
-26
lines changed

‎flink-filesystems/flink-azure-fs-hadoop/pom.xml‎

Lines changed: 11 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -22,8 +22,9 @@ under the License.
2222

2323
<parent>
2424
<groupId>org.apache.flink</groupId>
25-
<artifactId>flink-filesystems</artifactId>
25+
<artifactId>flink-hadoop34-fs-base</artifactId>
2626
<version>2.2-SNAPSHOT</version>
27+
<relativePath>../flink-hadoop34-fs-base</relativePath>
2728
</parent>
2829

2930
<artifactId>flink-azure-fs-hadoop</artifactId>
@@ -35,7 +36,7 @@ under the License.
3536
<properties>
3637
<fs.azure.sdk.version>1.16.0</fs.azure.sdk.version>
3738
<fs.jackson.core.version>2.9.4</fs.jackson.core.version>
38-
<jetty.version>9.3.24.v20180605</jetty.version>
39+
<jetty.version>9.4.53.v20231009</jetty.version>
3940
</properties>
4041

4142
<dependencies>
@@ -49,21 +50,24 @@ under the License.
4950

5051
<dependency>
5152
<groupId>org.apache.flink</groupId>
52-
<artifactId>flink-fs-hadoop-shaded</artifactId>
53+
<artifactId>flink-fs-hadoop34-shaded</artifactId>
5354
<version>${project.version}</version>
5455
<optional>${flink.markBundledAsOptional}</optional>
5556
</dependency>
5657

58+
<!-- Hadoop's file system abstraction (bundled) -->
59+
<dependency>
60+
<groupId>org.apache.hadoop</groupId>
61+
<artifactId>hadoop-common</artifactId>
62+
<optional>${flink.markBundledAsOptional}</optional>
63+
</dependency>
64+
5765
<dependency>
5866
<groupId>org.apache.hadoop</groupId>
5967
<artifactId>hadoop-azure</artifactId>
6068
<version>${fs.hadoopshaded.version}</version>
6169
<optional>${flink.markBundledAsOptional}</optional>
6270
<exclusions>
63-
<exclusion>
64-
<groupId>org.apache.hadoop</groupId>
65-
<artifactId>hadoop-common</artifactId>
66-
</exclusion>
6771
<exclusion>
6872
<groupId>ch.qos.reload4j</groupId>
6973
<artifactId>reload4j</artifactId>

‎flink-filesystems/flink-azure-fs-hadoop/src/main/resources/META-INF/NOTICE‎

Lines changed: 44 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -6,27 +6,56 @@ The Apache Software Foundation (http://www.apache.org/).
66

77
This project bundles the following dependencies under the Apache Software License 2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt)
88

9-
- com.fasterxml.jackson.core:jackson-core:2.18.2
10-
- com.google.guava:guava:20.0
119
- commons-codec:commons-codec:1.15
10+
- commons-io:commons-io:2.15.1
1211
- commons-logging:commons-logging:1.1.3
13-
- org.apache.hadoop.thirdparty:hadoop-shaded-guava:1.1.1
14-
- org.apache.hadoop:hadoop-azure:3.3.4
12+
- commons-collections:commons-collections:3.2.2
13+
- com.github.pjfanning:jersey-json:1.22.0
14+
- com.sun.xml.bind:jaxb-impl:2.2.3-1
15+
- javax.xml.bind:jaxb-api:2.3.1
16+
- javax.activation:javax.activation-api:1.2.0
17+
- org.codehaus.jettison:jettison:1.5.4
18+
- commons-beanutils:commons-beanutils:1.9.4
19+
- org.apache.commons:commons-configuration2:2.10.1
20+
- org.apache.commons:commons-lang3:3.12.0
21+
- org.apache.commons:commons-text:1.10.0
22+
- com.google.re2j:re2j:1.1
23+
- org.apache.hadoop:hadoop-auth:3.4.1
24+
- org.apache.kerby:kerb-util:2.0.3
25+
- org.apache.kerby:kerby-config:2.0.3
26+
- org.apache.kerby:kerb-crypto:2.0.3
27+
- io.netty:netty-handler:4.1.100.Final
28+
- io.netty:netty-common:4.1.100.Final
29+
- io.netty:netty-resolver:4.1.100.Final
30+
- io.netty:netty-buffer:4.1.100.Final
31+
- io.netty:netty-transport:4.1.100.Final
32+
- io.netty:netty-transport-native-unix-common:4.1.100.Final
33+
- io.netty:netty-codec:4.1.100.Final
34+
- io.netty:netty-transport-native-epoll:4.1.100.Final
35+
- io.netty:netty-transport-classes-epoll:4.1.100.Final
36+
- io.dropwizard.metrics:metrics-core:3.2.4
37+
- org.apache.commons:commons-compress:1.26.0
38+
- org.bouncycastle:bcprov-jdk18on:1.78.1
39+
- org.apache.kerby:kerb-core:2.0.3
40+
- org.apache.kerby:kerby-pkix:2.0.3
41+
- org.apache.kerby:kerby-asn1:2.0.3
42+
- org.apache.kerby:kerby-util:2.0.3
43+
- com.fasterxml.jackson.core:jackson-databind:2.18.2
44+
- com.fasterxml.jackson.core:jackson-annotations:2.18.2
45+
- com.fasterxml.jackson.core:jackson-core:2.18.2
46+
- org.codehaus.woodstox:stax2-api:4.2.1
47+
- com.fasterxml.woodstox:woodstox-core:5.4.0
48+
- dnsjava:dnsjava:3.6.1
49+
- org.xerial.snappy:snappy-java:1.1.10.7
50+
- org.apache.hadoop.thirdparty:hadoop-shaded-guava:1.3.0
51+
- org.apache.hadoop:hadoop-azure:3.4.1
1552
- org.apache.httpcomponents:httpclient:4.5.13
1653
- org.apache.httpcomponents:httpcore:4.4.14
17-
- org.codehaus.jackson:jackson-core-asl:1.9.13
18-
- org.codehaus.jackson:jackson-mapper-asl:1.9.14.jdk17-redhat-00001
19-
- org.eclipse.jetty:jetty-util-ajax:9.3.24.v20180605
20-
- org.eclipse.jetty:jetty-util:9.3.24.v20180605
21-
- org.wildfly.openssl:wildfly-openssl:1.0.7.Final
54+
- org.eclipse.jetty:jetty-util-ajax:9.4.53.v20231009
55+
- org.eclipse.jetty:jetty-util:9.4.53.v20231009
56+
- org.wildfly.openssl:wildfly-openssl:1.1.3.Final
2257

2358
This project bundles the following dependencies under the MIT (https://opensource.org/licenses/MIT)
2459

2560
- com.microsoft.azure:azure-keyvault-core:1.0.0
2661
- com.microsoft.azure:azure-storage:7.0.1
27-
28-
The bundled Apache Hadoop Relocated (Shaded) Third-party Miscellaneous Libs
29-
org.apache.hadoop.thirdparty:hadoop-shaded-guava dependency bundles the following dependencies under
30-
the Apache Software License 2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt)
31-
32-
- com.google.guava:guava:30.1.1-jre

‎flink-filesystems/flink-azure-fs-hadoop/src/main/resources/META-INF/licenses/LICENSE-re2j‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
This is a work derived from Russ Cox's RE2 in Go, whose license
2+
http://golang.org/LICENSE is as follows:
3+
4+
Copyright (c) 2009 The Go Authors. All rights reserved.
5+
6+
Redistribution and use in source and binary forms, with or without
7+
modification, are permitted provided that the following conditions are
8+
met:
9+
10+
* Redistributions of source code must retain the above copyright
11+
notice, this list of conditions and the following disclaimer.
12+
13+
* Redistributions in binary form must reproduce the above copyright
14+
notice, this list of conditions and the following disclaimer in
15+
the documentation and/or other materials provided with the
16+
distribution.
17+
18+
* Neither the name of Google Inc. nor the names of its contributors
19+
may be used to endorse or promote products derived from this
20+
software without specific prior written permission.
21+
22+
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
23+
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
24+
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
25+
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
26+
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
27+
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
28+
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
29+
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
30+
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
31+
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
32+
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

‎flink-filesystems/flink-azure-fs-hadoop/src/main/resources/META-INF/licenses/LICENSE-stax2api‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
Copyright woodstox stax2api contributors.
2+
3+
Redistribution and use in source and binary forms, with or without modification,
4+
are permitted provided that the following conditions are met:
5+
6+
1. Redistributions of source code must retain the above copyright notice,
7+
this list of conditions and the following disclaimer.
8+
9+
2. Redistributions in binary form must reproduce the above copyright notice,
10+
this list of conditions and the following disclaimer in the documentation
11+
and/or other materials provided with the distribution.
12+
13+
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
14+
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
15+
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
16+
IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
17+
INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
18+
BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA,
19+
OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
20+
WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
21+
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
22+
POSSIBILITY OF SUCH DAMAGE.

‎flink-filesystems/flink-fs-hadoop34-shaded/pom.xml‎

Lines changed: 131 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,131 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<!--
3+
Licensed to the Apache Software Foundation (ASF) under one
4+
or more contributor license agreements. See the NOTICE file
5+
distributed with this work for additional information
6+
regarding copyright ownership. The ASF licenses this file
7+
to you under the Apache License, Version 2.0 (the
8+
"License"); you may not use this file except in compliance
9+
with the License. You may obtain a copy of the License at
10+
http://www.apache.org/licenses/LICENSE-2.0
11+
Unless required by applicable law or agreed to in writing,
12+
software distributed under the License is distributed on an
13+
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
KIND, either express or implied. See the License for the
15+
specific language governing permissions and limitations
16+
under the License.
17+
-->
18+
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
19+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
20+
21+
<modelVersion>4.0.0</modelVersion>
22+
23+
<parent>
24+
<groupId>org.apache.flink</groupId>
25+
<artifactId>flink-hadoop34-fs-base</artifactId>
26+
<version>2.2-SNAPSHOT</version>
27+
<relativePath>../flink-hadoop34-fs-base</relativePath>
28+
</parent>
29+
30+
<artifactId>flink-fs-hadoop34-shaded</artifactId>
31+
<name>Flink : FileSystems : Hadoop FS 3.4 shaded</name>
32+
33+
<packaging>jar</packaging>
34+
35+
<properties>
36+
<japicmp.skip>true</japicmp.skip>
37+
</properties>
38+
39+
<dependencies>
40+
<!-- The Hadoop file system abstraction -->
41+
<dependency>
42+
<groupId>org.apache.hadoop</groupId>
43+
<artifactId>hadoop-common</artifactId>
44+
<optional>${flink.markBundledAsOptional}</optional>
45+
</dependency>
46+
</dependencies>
47+
48+
<build>
49+
50+
<!-- this is merely an intermediate build artifact and should not be -->
51+
<!-- deployed to maven central -->
52+
<plugins>
53+
<plugin>
54+
<groupId>org.apache.maven.plugins</groupId>
55+
<artifactId>maven-deploy-plugin</artifactId>
56+
<configuration>
57+
<skip>true</skip>
58+
</configuration>
59+
</plugin>
60+
61+
<!-- publish the core-site.xml for tests -->
62+
<plugin>
63+
<groupId>org.apache.maven.plugins</groupId>
64+
<artifactId>maven-jar-plugin</artifactId>
65+
<executions>
66+
<execution>
67+
<goals>
68+
<goal>test-jar</goal>
69+
</goals>
70+
</execution>
71+
</executions>
72+
</plugin>
73+
74+
<!-- relocate all dependencies to hide them -->
75+
<plugin>
76+
<groupId>org.apache.maven.plugins</groupId>
77+
<artifactId>maven-shade-plugin</artifactId>
78+
<executions>
79+
<execution>
80+
<id>shade-flink</id>
81+
<phase>package</phase>
82+
<goals>
83+
<goal>shade</goal>
84+
</goals>
85+
<configuration>
86+
<artifactSet>
87+
<includes>
88+
<include>*:*</include>
89+
</includes>
90+
</artifactSet>
91+
<relocations>
92+
<!-- we shade only the parts that are internal to Hadoop and not used / exposed downstream -->
93+
<relocation>
94+
<pattern>com.google.re2j</pattern>
95+
<shadedPattern>org.apache.flink.fs.shaded.hadoop3.com.google.re2j</shadedPattern>
96+
</relocation>
97+
<relocation>
98+
<pattern>org.apache.htrace</pattern>
99+
<shadedPattern>org.apache.flink.fs.shaded.hadoop3.org.apache.htrace</shadedPattern>
100+
</relocation>
101+
<relocation>
102+
<pattern>com.fasterxml</pattern>
103+
<shadedPattern>org.apache.flink.fs.shaded.hadoop3.com.fasterxml</shadedPattern>
104+
</relocation>
105+
<relocation>
106+
<pattern>org.codehaus</pattern>
107+
<shadedPattern>org.apache.flink.fs.shaded.hadoop3.org.codehaus</shadedPattern>
108+
</relocation>
109+
<relocation>
110+
<pattern>com.ctc</pattern>
111+
<shadedPattern>org.apache.flink.fs.shaded.hadoop3.com.ctc</shadedPattern>
112+
</relocation>
113+
</relocations>
114+
<filters>
115+
<filter>
116+
<artifact>*</artifact>
117+
<excludes>
118+
<exclude>properties.dtd</exclude>
119+
<exclude>PropertyList-1.0.dtd</exclude>
120+
<exclude>META-INF/services/javax.xml.stream.*</exclude>
121+
<exclude>META-INF/LICENSE.txt</exclude>
122+
</excludes>
123+
</filter>
124+
</filters>
125+
</configuration>
126+
</execution>
127+
</executions>
128+
</plugin>
129+
</plugins>
130+
</build>
131+
</project>

0 commit comments

Comments
 (0)