Skip to content

Commit b62096d

Browse files
mohsenrezaeithemohsenrezaeithe
committed
implemented Hadoop 3.4 module + migrated Azure to it
1 parent 619791a commit b62096d

File tree

8 files changed

+715
-24
lines changed

8 files changed

+715
-24
lines changed

‎flink-filesystems/flink-azure-fs-hadoop/pom.xml

Lines changed: 105 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -22,8 +22,9 @@ under the License.
2222

2323
<parent>
2424
<groupId>org.apache.flink</groupId>
25-
<artifactId>flink-filesystems</artifactId>
25+
<artifactId>flink-hadoop34-fs-base</artifactId>
2626
<version>2.2-SNAPSHOT</version>
27+
<relativePath>../flink-hadoop34-fs-base</relativePath>
2728
</parent>
2829

2930
<artifactId>flink-azure-fs-hadoop</artifactId>
@@ -35,7 +36,7 @@ under the License.
3536
<properties>
3637
<fs.azure.sdk.version>1.16.0</fs.azure.sdk.version>
3738
<fs.jackson.core.version>2.9.4</fs.jackson.core.version>
38-
<jetty.version>9.3.24.v20180605</jetty.version>
39+
<jetty.version>9.4.53.v20231009</jetty.version>
3940
</properties>
4041

4142
<dependencies>
@@ -49,21 +50,33 @@ under the License.
4950

5051
<dependency>
5152
<groupId>org.apache.flink</groupId>
52-
<artifactId>flink-fs-hadoop-shaded</artifactId>
53+
<artifactId>flink-fs-hadoop34-shaded</artifactId>
5354
<version>${project.version}</version>
5455
<optional>${flink.markBundledAsOptional}</optional>
5556
</dependency>
5657

58+
<!-- Hadoop's file system abstraction (bundled) -->
59+
<dependency>
60+
<groupId>org.apache.hadoop</groupId>
61+
<artifactId>hadoop-common</artifactId>
62+
<optional>${flink.markBundledAsOptional}</optional>
63+
</dependency>
64+
65+
<dependency>
66+
<!-- Hadoop requires jaxb-api for javax.xml.bind.JAXBException -->
67+
<groupId>javax.xml.bind</groupId>
68+
<artifactId>jaxb-api</artifactId>
69+
<version>${jaxb.api.version}</version>
70+
<!-- packaged as an optional dependency that is only accessible on Java 11+ -->
71+
<scope>provided</scope>
72+
</dependency>
73+
5774
<dependency>
5875
<groupId>org.apache.hadoop</groupId>
5976
<artifactId>hadoop-azure</artifactId>
6077
<version>${fs.hadoopshaded.version}</version>
6178
<optional>${flink.markBundledAsOptional}</optional>
6279
<exclusions>
63-
<exclusion>
64-
<groupId>org.apache.hadoop</groupId>
65-
<artifactId>hadoop-common</artifactId>
66-
</exclusion>
6780
<exclusion>
6881
<groupId>ch.qos.reload4j</groupId>
6982
<artifactId>reload4j</artifactId>
@@ -81,6 +94,12 @@ under the License.
8194
<artifactId>azure</artifactId>
8295
<version>${fs.azure.sdk.version}</version>
8396
<scope>test</scope>
97+
<exclusions>
98+
<exclusion>
99+
<groupId>com.google.guava</groupId>
100+
<artifactId>guava</artifactId>
101+
</exclusion>
102+
</exclusions>
84103
</dependency>
85104

86105
</dependencies>
@@ -98,6 +117,76 @@ under the License.
98117

99118
<build>
100119
<plugins>
120+
<plugin>
121+
<groupId>org.apache.maven.plugins</groupId>
122+
<artifactId>maven-jar-plugin</artifactId>
123+
<executions>
124+
<execution>
125+
<goals>
126+
<goal>jar</goal>
127+
</goals>
128+
</execution>
129+
</executions>
130+
<configuration>
131+
<archive>
132+
<manifestEntries>
133+
<!-- jaxb-api is packaged as an optional dependency that is only accessible on Java 11 -->
134+
<Multi-Release>true</Multi-Release>
135+
</manifestEntries>
136+
</archive>
137+
</configuration>
138+
</plugin>
139+
140+
<plugin>
141+
<groupId>org.apache.maven.plugins</groupId>
142+
<artifactId>maven-dependency-plugin</artifactId>
143+
<executions>
144+
<execution>
145+
<id>copy-javax-jars</id>
146+
<phase>process-resources</phase>
147+
<goals>
148+
<goal>copy</goal>
149+
</goals>
150+
</execution>
151+
</executions>
152+
<configuration>
153+
<artifactItems>
154+
<artifactItem>
155+
<groupId>javax.xml.bind</groupId>
156+
<artifactId>jaxb-api</artifactId>
157+
<version>${jaxb.api.version}</version>
158+
<type>jar</type>
159+
<overWrite>true</overWrite>
160+
</artifactItem>
161+
</artifactItems>
162+
<outputDirectory>${project.build.directory}/temporary</outputDirectory>
163+
</configuration>
164+
</plugin>
165+
166+
<plugin>
167+
<groupId>org.apache.maven.plugins</groupId>
168+
<artifactId>maven-antrun-plugin</artifactId>
169+
<executions>
170+
<execution>
171+
<id>unpack-javax-libraries</id>
172+
<phase>process-resources</phase>
173+
<goals>
174+
<goal>run</goal>
175+
</goals>
176+
<configuration>
177+
<target>
178+
<echo message="unpacking javax jars"/>
179+
<unzip dest="${project.build.directory}/classes/META-INF/versions/11">
180+
<fileset dir="${project.build.directory}/temporary">
181+
<include name="*"/>
182+
</fileset>
183+
</unzip>
184+
</target>
185+
</configuration>
186+
</execution>
187+
</executions>
188+
</plugin>
189+
101190
<!-- Relocate all Azure related classes -->
102191
<plugin>
103192
<groupId>org.apache.maven.plugins</groupId>
@@ -137,6 +226,15 @@ under the License.
137226
<exclude>META-INF/LICENSE.txt</exclude>
138227
</excludes>
139228
</filter>
229+
<filter>
230+
<artifact>*</artifact>
231+
<excludes>
232+
<exclude>properties.dtd</exclude>
233+
<exclude>PropertyList-1.0.dtd</exclude>
234+
<exclude>META-INF/services/javax.xml.stream.*</exclude>
235+
<exclude>META-INF/LICENSE.txt</exclude>
236+
</excludes>
237+
</filter>
140238
</filters>
141239
</configuration>
142240
</execution>

‎flink-filesystems/flink-azure-fs-hadoop/src/main/resources/META-INF/NOTICE

Lines changed: 64 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -6,27 +6,78 @@ The Apache Software Foundation (http://www.apache.org/).
66

77
This project bundles the following dependencies under the Apache Software License 2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt)
88

9-
- com.fasterxml.jackson.core:jackson-core:2.18.2
10-
- com.google.guava:guava:20.0
9+
- org.apache.hadoop:hadoop-common:3.4.1
10+
- org.apache.hadoop:hadoop-annotations:3.4.1
11+
- org.apache.hadoop.thirdparty:hadoop-shaded-protobuf_3_25:1.3.0
12+
- com.google.guava:guava:27.0-jre
13+
- com.google.guava:failureaccess:1.0
14+
- com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
15+
- org.checkerframework:checker-qual:2.5.2
16+
- com.google.j2objc:j2objc-annotations:1.1
17+
- org.codehaus.mojo:animal-sniffer-annotations:1.17
1118
- commons-codec:commons-codec:1.15
19+
- commons-io:commons-io:2.15.1
1220
- commons-logging:commons-logging:1.1.3
13-
- org.apache.hadoop.thirdparty:hadoop-shaded-guava:1.1.1
14-
- org.apache.hadoop:hadoop-azure:3.3.4
21+
- commons-collections:commons-collections:3.2.2
22+
- com.github.pjfanning:jersey-json:1.22.0
23+
- com.sun.xml.bind:jaxb-impl:2.2.3-1
24+
- javax.activation:javax.activation-api:1.2.0
25+
- org.codehaus.jettison:jettison:1.5.4
26+
- commons-beanutils:commons-beanutils:1.9.4
27+
- org.apache.commons:commons-configuration2:2.10.1
28+
- org.apache.commons:commons-lang3:3.12.0
29+
- org.apache.commons:commons-text:1.10.0
30+
- org.apache.hadoop:hadoop-auth:3.4.1
31+
- org.apache.kerby:kerb-util:2.0.3
32+
- org.apache.kerby:kerby-config:2.0.3
33+
- org.apache.kerby:kerb-crypto:2.0.3
34+
- io.netty:netty-handler:4.1.100.Final
35+
- io.netty:netty-common:4.1.100.Final
36+
- io.netty:netty-resolver:4.1.100.Final
37+
- io.netty:netty-buffer:4.1.100.Final
38+
- io.netty:netty-transport:4.1.100.Final
39+
- io.netty:netty-transport-native-unix-common:4.1.100.Final
40+
- io.netty:netty-codec:4.1.100.Final
41+
- io.netty:netty-transport-native-epoll:4.1.100.Final
42+
- io.netty:netty-transport-classes-epoll:4.1.100.Final
43+
- io.dropwizard.metrics:metrics-core:3.2.4
44+
- org.apache.commons:commons-compress:1.26.0
45+
- org.bouncycastle:bcprov-jdk18on:1.78.1
46+
- org.apache.kerby:kerb-core:2.0.3
47+
- org.apache.kerby:kerby-pkix:2.0.3
48+
- org.apache.kerby:kerby-asn1:2.0.3
49+
- org.apache.kerby:kerby-util:2.0.3
50+
- com.fasterxml.jackson.core:jackson-databind:2.18.2
51+
- com.fasterxml.jackson.core:jackson-annotations:2.18.2
52+
- com.fasterxml.jackson.core:jackson-core:2.18.2
53+
- org.codehaus.woodstox:stax2-api:4.2.1
54+
- com.fasterxml.woodstox:woodstox-core:5.4.0
55+
- dnsjava:dnsjava:3.6.1
56+
- org.xerial.snappy:snappy-java:1.1.10.7
57+
- org.apache.hadoop.thirdparty:hadoop-shaded-guava:1.3.0
58+
- org.apache.hadoop:hadoop-azure:3.4.1
1559
- org.apache.httpcomponents:httpclient:4.5.13
1660
- org.apache.httpcomponents:httpcore:4.4.14
17-
- org.codehaus.jackson:jackson-core-asl:1.9.13
18-
- org.codehaus.jackson:jackson-mapper-asl:1.9.14.jdk17-redhat-00001
19-
- org.eclipse.jetty:jetty-util-ajax:9.3.24.v20180605
20-
- org.eclipse.jetty:jetty-util:9.3.24.v20180605
21-
- org.wildfly.openssl:wildfly-openssl:1.0.7.Final
61+
- org.eclipse.jetty:jetty-util-ajax:9.4.53.v20231009
62+
- org.eclipse.jetty:jetty-util:9.4.53.v20231009
63+
- org.wildfly.openssl:wildfly-openssl:1.1.3.Final
2264

2365
This project bundles the following dependencies under the MIT (https://opensource.org/licenses/MIT)
2466

2567
- com.microsoft.azure:azure-keyvault-core:1.0.0
2668
- com.microsoft.azure:azure-storage:7.0.1
2769

28-
The bundled Apache Hadoop Relocated (Shaded) Third-party Miscellaneous Libs
29-
org.apache.hadoop.thirdparty:hadoop-shaded-guava dependency bundles the following dependencies under
30-
the Apache Software License 2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt)
70+
This project bundles the following dependencies under the CDDL 1.1 license.
71+
See bundled license files for details.
72+
73+
- javax.xml.bind:jaxb-api:2.3.1
74+
75+
This project bundles the following dependencies under the Go License (https://golang.org/LICENSE).
76+
See bundled license files for details.
77+
78+
- com.google.re2j:re2j:1.1
79+
80+
This project bundles the following dependencies under BSD License (https://opensource.org/licenses/bsd-license.php).
81+
See bundled license files for details.
3182

32-
- com.google.guava:guava:30.1.1-jre
83+
- org.codehaus.woodstox:stax2-api:4.2.1 (https://github.com/FasterXML/stax2-api/tree/stax2-api-4.2.1)

‎flink-filesystems/flink-azure-fs-hadoop/src/main/resources/META-INF/licenses/LICENSE-re2j

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
This is a work derived from Russ Cox's RE2 in Go, whose license
2+
http://golang.org/LICENSE is as follows:
3+
4+
Copyright (c) 2009 The Go Authors. All rights reserved.
5+
6+
Redistribution and use in source and binary forms, with or without
7+
modification, are permitted provided that the following conditions are
8+
met:
9+
10+
* Redistributions of source code must retain the above copyright
11+
notice, this list of conditions and the following disclaimer.
12+
13+
* Redistributions in binary form must reproduce the above copyright
14+
notice, this list of conditions and the following disclaimer in
15+
the documentation and/or other materials provided with the
16+
distribution.
17+
18+
* Neither the name of Google Inc. nor the names of its contributors
19+
may be used to endorse or promote products derived from this
20+
software without specific prior written permission.
21+
22+
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
23+
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
24+
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
25+
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
26+
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
27+
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
28+
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
29+
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
30+
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
31+
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
32+
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

‎flink-filesystems/flink-azure-fs-hadoop/src/main/resources/META-INF/licenses/LICENSE-stax2api

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
Copyright woodstox stax2api contributors.
2+
3+
Redistribution and use in source and binary forms, with or without modification,
4+
are permitted provided that the following conditions are met:
5+
6+
1. Redistributions of source code must retain the above copyright notice,
7+
this list of conditions and the following disclaimer.
8+
9+
2. Redistributions in binary form must reproduce the above copyright notice,
10+
this list of conditions and the following disclaimer in the documentation
11+
and/or other materials provided with the distribution.
12+
13+
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
14+
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
15+
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
16+
IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
17+
INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
18+
BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA,
19+
OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
20+
WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
21+
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
22+
POSSIBILITY OF SUCH DAMAGE.

0 commit comments

Comments
 (0)