Skip to content

Commit 501ac62

Browse files
mohsenrezaeithemohsenrezaeithe
committed
implemented Hadoop 3.4 module + migrated Azure to it
1 parent e71204f commit 501ac62

File tree

5 files changed

+373
-26
lines changed

5 files changed

+373
-26
lines changed

‎flink-filesystems/flink-azure-fs-hadoop/pom.xml‎

Lines changed: 11 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -22,8 +22,9 @@ under the License.
2222

2323
<parent>
2424
<groupId>org.apache.flink</groupId>
25-
<artifactId>flink-filesystems</artifactId>
25+
<artifactId>flink-hadoop34-fs-base</artifactId>
2626
<version>2.2-SNAPSHOT</version>
27+
<relativePath>../flink-hadoop34-fs-base</relativePath>
2728
</parent>
2829

2930
<artifactId>flink-azure-fs-hadoop</artifactId>
@@ -35,7 +36,7 @@ under the License.
3536
<properties>
3637
<fs.azure.sdk.version>1.16.0</fs.azure.sdk.version>
3738
<fs.jackson.core.version>2.9.4</fs.jackson.core.version>
38-
<jetty.version>9.3.24.v20180605</jetty.version>
39+
<jetty.version>9.4.53.v20231009</jetty.version>
3940
</properties>
4041

4142
<dependencies>
@@ -49,21 +50,24 @@ under the License.
4950

5051
<dependency>
5152
<groupId>org.apache.flink</groupId>
52-
<artifactId>flink-fs-hadoop-shaded</artifactId>
53+
<artifactId>flink-fs-hadoop34-shaded</artifactId>
5354
<version>${project.version}</version>
5455
<optional>${flink.markBundledAsOptional}</optional>
5556
</dependency>
5657

58+
<!-- Hadoop's file system abstraction (bundled) -->
59+
<dependency>
60+
<groupId>org.apache.hadoop</groupId>
61+
<artifactId>hadoop-common</artifactId>
62+
<optional>${flink.markBundledAsOptional}</optional>
63+
</dependency>
64+
5765
<dependency>
5866
<groupId>org.apache.hadoop</groupId>
5967
<artifactId>hadoop-azure</artifactId>
6068
<version>${fs.hadoopshaded.version}</version>
6169
<optional>${flink.markBundledAsOptional}</optional>
6270
<exclusions>
63-
<exclusion>
64-
<groupId>org.apache.hadoop</groupId>
65-
<artifactId>hadoop-common</artifactId>
66-
</exclusion>
6771
<exclusion>
6872
<groupId>ch.qos.reload4j</groupId>
6973
<artifactId>reload4j</artifactId>

‎flink-filesystems/flink-azure-fs-hadoop/src/main/resources/META-INF/NOTICE‎

Lines changed: 5 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -6,27 +6,17 @@ The Apache Software Foundation (http://www.apache.org/).
66

77
This project bundles the following dependencies under the Apache Software License 2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt)
88

9-
- com.fasterxml.jackson.core:jackson-core:2.18.2
10-
- com.google.guava:guava:20.0
119
- commons-codec:commons-codec:1.15
1210
- commons-logging:commons-logging:1.1.3
13-
- org.apache.hadoop.thirdparty:hadoop-shaded-guava:1.1.1
14-
- org.apache.hadoop:hadoop-azure:3.3.4
11+
- org.apache.hadoop.thirdparty:hadoop-shaded-guava:1.3.0
12+
- org.apache.hadoop:hadoop-azure:3.4.1
1513
- org.apache.httpcomponents:httpclient:4.5.13
1614
- org.apache.httpcomponents:httpcore:4.4.14
17-
- org.codehaus.jackson:jackson-core-asl:1.9.13
18-
- org.codehaus.jackson:jackson-mapper-asl:1.9.14.jdk17-redhat-00001
19-
- org.eclipse.jetty:jetty-util-ajax:9.3.24.v20180605
20-
- org.eclipse.jetty:jetty-util:9.3.24.v20180605
21-
- org.wildfly.openssl:wildfly-openssl:1.0.7.Final
15+
- org.eclipse.jetty:jetty-util-ajax:9.4.53.v20231009
16+
- org.eclipse.jetty:jetty-util:9.4.53.v20231009
17+
- org.wildfly.openssl:wildfly-openssl:1.1.3.Final
2218

2319
This project bundles the following dependencies under the MIT (https://opensource.org/licenses/MIT)
2420

2521
- com.microsoft.azure:azure-keyvault-core:1.0.0
2622
- com.microsoft.azure:azure-storage:7.0.1
27-
28-
The bundled Apache Hadoop Relocated (Shaded) Third-party Miscellaneous Libs
29-
org.apache.hadoop.thirdparty:hadoop-shaded-guava dependency bundles the following dependencies under
30-
the Apache Software License 2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt)
31-
32-
- com.google.guava:guava:30.1.1-jre

‎flink-filesystems/flink-fs-hadoop34-shaded/pom.xml‎

Lines changed: 131 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,131 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<!--
3+
Licensed to the Apache Software Foundation (ASF) under one
4+
or more contributor license agreements. See the NOTICE file
5+
distributed with this work for additional information
6+
regarding copyright ownership. The ASF licenses this file
7+
to you under the Apache License, Version 2.0 (the
8+
"License"); you may not use this file except in compliance
9+
with the License. You may obtain a copy of the License at
10+
http://www.apache.org/licenses/LICENSE-2.0
11+
Unless required by applicable law or agreed to in writing,
12+
software distributed under the License is distributed on an
13+
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
KIND, either express or implied. See the License for the
15+
specific language governing permissions and limitations
16+
under the License.
17+
-->
18+
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
19+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
20+
21+
<modelVersion>4.0.0</modelVersion>
22+
23+
<parent>
24+
<groupId>org.apache.flink</groupId>
25+
<artifactId>flink-hadoop34-fs-base</artifactId>
26+
<version>2.2-SNAPSHOT</version>
27+
<relativePath>../flink-hadoop34-fs-base</relativePath>
28+
</parent>
29+
30+
<artifactId>flink-fs-hadoop34-shaded</artifactId>
31+
<name>Flink : FileSystems : Hadoop FS 3.4 shaded</name>
32+
33+
<packaging>jar</packaging>
34+
35+
<properties>
36+
<japicmp.skip>true</japicmp.skip>
37+
</properties>
38+
39+
<dependencies>
40+
<!-- The Hadoop file system abstraction -->
41+
<dependency>
42+
<groupId>org.apache.hadoop</groupId>
43+
<artifactId>hadoop-common</artifactId>
44+
<optional>${flink.markBundledAsOptional}</optional>
45+
</dependency>
46+
</dependencies>
47+
48+
<build>
49+
50+
<!-- this is merely an intermediate build artifact and should not be -->
51+
<!-- deployed to maven central -->
52+
<plugins>
53+
<plugin>
54+
<groupId>org.apache.maven.plugins</groupId>
55+
<artifactId>maven-deploy-plugin</artifactId>
56+
<configuration>
57+
<skip>true</skip>
58+
</configuration>
59+
</plugin>
60+
61+
<!-- publish the core-site.xml for tests -->
62+
<plugin>
63+
<groupId>org.apache.maven.plugins</groupId>
64+
<artifactId>maven-jar-plugin</artifactId>
65+
<executions>
66+
<execution>
67+
<goals>
68+
<goal>test-jar</goal>
69+
</goals>
70+
</execution>
71+
</executions>
72+
</plugin>
73+
74+
<!-- relocate all dependencies to hide them -->
75+
<plugin>
76+
<groupId>org.apache.maven.plugins</groupId>
77+
<artifactId>maven-shade-plugin</artifactId>
78+
<executions>
79+
<execution>
80+
<id>shade-flink</id>
81+
<phase>package</phase>
82+
<goals>
83+
<goal>shade</goal>
84+
</goals>
85+
<configuration>
86+
<artifactSet>
87+
<includes>
88+
<include>*:*</include>
89+
</includes>
90+
</artifactSet>
91+
<relocations>
92+
<!-- we shade only the parts that are internal to Hadoop and not used / exposed downstream -->
93+
<relocation>
94+
<pattern>com.google.re2j</pattern>
95+
<shadedPattern>org.apache.flink.fs.shaded.hadoop3.com.google.re2j</shadedPattern>
96+
</relocation>
97+
<relocation>
98+
<pattern>org.apache.htrace</pattern>
99+
<shadedPattern>org.apache.flink.fs.shaded.hadoop3.org.apache.htrace</shadedPattern>
100+
</relocation>
101+
<relocation>
102+
<pattern>com.fasterxml</pattern>
103+
<shadedPattern>org.apache.flink.fs.shaded.hadoop3.com.fasterxml</shadedPattern>
104+
</relocation>
105+
<relocation>
106+
<pattern>org.codehaus</pattern>
107+
<shadedPattern>org.apache.flink.fs.shaded.hadoop3.org.codehaus</shadedPattern>
108+
</relocation>
109+
<relocation>
110+
<pattern>com.ctc</pattern>
111+
<shadedPattern>org.apache.flink.fs.shaded.hadoop3.com.ctc</shadedPattern>
112+
</relocation>
113+
</relocations>
114+
<filters>
115+
<filter>
116+
<artifact>*</artifact>
117+
<excludes>
118+
<exclude>properties.dtd</exclude>
119+
<exclude>PropertyList-1.0.dtd</exclude>
120+
<exclude>META-INF/services/javax.xml.stream.*</exclude>
121+
<exclude>META-INF/LICENSE.txt</exclude>
122+
</excludes>
123+
</filter>
124+
</filters>
125+
</configuration>
126+
</execution>
127+
</executions>
128+
</plugin>
129+
</plugins>
130+
</build>
131+
</project>

0 commit comments

Comments
 (0)