From 78c1bc49fff068fe1d61d6deabee2aa4d6fcb6d1 Mon Sep 17 00:00:00 2001 From: Yukihiro Okada Date: Wed, 27 May 2020 23:40:30 +0900 Subject: [PATCH 1/7] Minor upgrade for 0.4 --- build.gradle | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/build.gradle b/build.gradle index 4e63ce5..36a4c1b 100644 --- a/build.gradle +++ b/build.gradle @@ -20,14 +20,16 @@ configurations { runtime.exclude group: "org.slf4j", module: "slf4j-log4j12" } +def EMBULK_VERSION = "0.8.39" + version = "0.3.4" sourceCompatibility = 1.8 targetCompatibility = 1.8 dependencies { - compile "org.embulk:embulk-core:0.8.39" - provided "org.embulk:embulk-core:0.8.39" + compile "org.embulk:embulk-core:$EMBULK_VERSION" + provided "org.embulk:embulk-core:$EMBULK_VERSION" compile "org.scala-lang:scala-library:2.12.+" compile "org.apache.orc:orc:1.5.4" @@ -43,8 +45,8 @@ dependencies { // testCompile "junit:junit:4.+" testCompile 'org.hamcrest:hamcrest-core:1.3' testCompile 'org.testng:testng:6.14.2' - testCompile "org.embulk:embulk-core:0.8.39:tests" - testCompile "org.embulk:embulk-standards:0.8.39" + testCompile "org.embulk:embulk-core:$EMBULK_VERSION:tests" + testCompile "org.embulk:embulk-standards:$EMBULK_VERSION" } sourceSets { From d88f4032bc940c4819b4bd77ca39d2fb3101edad Mon Sep 17 00:00:00 2001 From: Yukihiro Okada Date: Wed, 27 May 2020 23:50:06 +0900 Subject: [PATCH 2/7] Upgrade embulk to 0.9.23 --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 36a4c1b..72e88d0 100644 --- a/build.gradle +++ b/build.gradle @@ -20,7 +20,7 @@ configurations { runtime.exclude group: "org.slf4j", module: "slf4j-log4j12" } -def EMBULK_VERSION = "0.8.39" +def EMBULK_VERSION = "0.9.23" version = "0.3.4" From 51bb97e1080bfbdd4284b9b908e49bf548cd3971 Mon Sep 17 00:00:00 2001 From: Yukihiro Okada Date: Thu, 28 May 2020 00:13:07 +0900 Subject: [PATCH 3/7] Upgrade orc to 1.5.10 --- build.gradle | 135 ++++++++++++++++++++++++++------------------------- 1 file changed, 69 insertions(+), 66 deletions(-) diff --git a/build.gradle b/build.gradle index 72e88d0..b4d8e6a 100644 --- a/build.gradle +++ b/build.gradle @@ -1,26 +1,28 @@ plugins { - id "com.jfrog.bintray" version "1.1" - id "com.github.jruby-gradle.base" version "1.5.0" - id "java" - id "scala" - id "checkstyle" + id "com.jfrog.bintray" version "1.1" + id "com.github.jruby-gradle.base" version "1.5.0" + id "java" + id "scala" + id "checkstyle" // id "com.github.spotbugs" version "3.0.1" - id "org.sonarqube" version "2.5" + id "org.sonarqube" version "2.5" } import com.github.jrubygradle.JRubyExec + repositories { - mavenCentral() - jcenter() - maven { - url "http://dl.bintray.com/embulk-input-s3/maven" - } + mavenCentral() + jcenter() + maven { + url "http://dl.bintray.com/embulk-input-s3/maven" + } } configurations { - provided - runtime.exclude group: "org.slf4j", module: "slf4j-log4j12" + provided + runtime.exclude group: "org.slf4j", module: "slf4j-log4j12" } def EMBULK_VERSION = "0.9.23" +def ORC_VERSION = "1.5.10" version = "0.3.4" @@ -28,87 +30,88 @@ sourceCompatibility = 1.8 targetCompatibility = 1.8 dependencies { - compile "org.embulk:embulk-core:$EMBULK_VERSION" - provided "org.embulk:embulk-core:$EMBULK_VERSION" - compile "org.scala-lang:scala-library:2.12.+" - - compile "org.apache.orc:orc:1.5.4" - compile "org.apache.orc:orc-core:1.5.4" - compile "org.apache.hadoop:hadoop-hdfs:2.7.5" - - compile 'org.embulk.input.s3:embulk-util-aws-credentials:0.2.8' - compile "com.amazonaws:aws-java-sdk-s3:1.10.33" - compile "org.apache.hadoop:hadoop-aws:2.7.5" - compile 'com.google.guava:guava:24.1-jre' - - testCompile 'org.jmockit:jmockit:1.38' - // testCompile "junit:junit:4.+" - testCompile 'org.hamcrest:hamcrest-core:1.3' - testCompile 'org.testng:testng:6.14.2' - testCompile "org.embulk:embulk-core:$EMBULK_VERSION:tests" - testCompile "org.embulk:embulk-standards:$EMBULK_VERSION" + compile "org.embulk:embulk-core:$EMBULK_VERSION" + provided "org.embulk:embulk-core:$EMBULK_VERSION" + compile "org.scala-lang:scala-library:2.12.+" + + compile "org.apache.orc:orc:$ORC_VERSION" + compile "org.apache.orc:orc-core:$ORC_VERSION" + compile "org.apache.hadoop:hadoop-hdfs:2.7.5" + + compile 'org.embulk.input.s3:embulk-util-aws-credentials:0.2.8' + compile "com.amazonaws:aws-java-sdk-s3:1.10.33" + compile "org.apache.hadoop:hadoop-aws:2.7.5" + compile 'com.google.guava:guava:24.1-jre' + + testCompile 'org.jmockit:jmockit:1.38' + // testCompile "junit:junit:4.+" + testCompile 'org.hamcrest:hamcrest-core:1.3' + testCompile 'org.testng:testng:6.14.2' + testCompile "org.embulk:embulk-core:$EMBULK_VERSION:tests" + testCompile "org.embulk:embulk-standards:$EMBULK_VERSION" } sourceSets { - main { - scala { - srcDirs = ['src/main/scala', 'src/main/java'] - } - java { - srcDirs = [] - } + main { + scala { + srcDirs = ['src/main/scala', 'src/main/java'] + } + java { + srcDirs = [] } + } } task classpath(type: Copy, dependsOn: ["jar"]) { - doFirst { file("classpath").deleteDir() } - from (configurations.runtime - configurations.provided + files(jar.archivePath)) - into "classpath" + doFirst { file("classpath").deleteDir() } + from(configurations.runtime - configurations.provided + files(jar.archivePath)) + into "classpath" } clean { delete "classpath" } checkstyle { - configFile = file("${project.rootDir}/config/checkstyle/checkstyle.xml") - toolVersion = '6.14.1' + configFile = file("${project.rootDir}/config/checkstyle/checkstyle.xml") + toolVersion = '6.14.1' } checkstyleMain { - configFile = file("${project.rootDir}/config/checkstyle/default.xml") - ignoreFailures = true + configFile = file("${project.rootDir}/config/checkstyle/default.xml") + ignoreFailures = true } checkstyleTest { - configFile = file("${project.rootDir}/config/checkstyle/default.xml") - ignoreFailures = true + configFile = file("${project.rootDir}/config/checkstyle/default.xml") + ignoreFailures = true } task checkstyle(type: Checkstyle) { - classpath = sourceSets.main.output + sourceSets.test.output - source = sourceSets.main.allJava + sourceSets.test.allJava + classpath = sourceSets.main.output + sourceSets.test.output + source = sourceSets.main.allJava + sourceSets.test.allJava } task gem(type: JRubyExec, dependsOn: ["gemspec", "classpath"]) { - jrubyArgs "-S" - script "gem" - scriptArgs "build", "${project.name}.gemspec" - doLast { ant.move(file: "${project.name}-${project.version}.gem", todir: "pkg") } + jrubyArgs "-S" + script "gem" + scriptArgs "build", "${project.name}.gemspec" + doLast { ant.move(file: "${project.name}-${project.version}.gem", todir: "pkg") } } task gemPush(type: JRubyExec, dependsOn: ["gem"]) { - jrubyArgs "-S" - script "gem" - scriptArgs "push", "pkg/${project.name}-${project.version}.gem" + jrubyArgs "-S" + script "gem" + scriptArgs "push", "pkg/${project.name}-${project.version}.gem" } task "package"(dependsOn: ["gemspec", "classpath"]) { - doLast { - println "> Build succeeded." - println "> You can run embulk with '-L ${file(".").absolutePath}' argument." - } + doLast { + println "> Build succeeded." + println "> You can run embulk with '-L ${file(".").absolutePath}' argument." + } } task gemspec { - ext.gemspecFile = file("${project.name}.gemspec") - inputs.file "build.gradle" - outputs.file gemspecFile - doLast { gemspecFile.write($/ + ext.gemspecFile = file("${project.name}.gemspec") + inputs.file "build.gradle" + outputs.file gemspecFile + doLast { + gemspecFile.write($/ Gem::Specification.new do |spec| spec.name = "${project.name}" spec.version = "${project.version}" @@ -128,6 +131,6 @@ Gem::Specification.new do |spec| spec.add_development_dependency 'rake', ['>= 10.0'] end /$) - } + } } clean { delete "${project.name}.gemspec" } From a328b566798936a80a851a9c577effcbe31b4d09 Mon Sep 17 00:00:00 2001 From: Yukihiro Okada Date: Thu, 28 May 2020 02:04:12 +0900 Subject: [PATCH 4/7] Upgrade dependency libraries - Upgrade hadoop-hdfs - Upgrade aws-java-sdk-s3 - etc From ed7f186710fa2a587a5bcf5f20c8fc35a66af15e Mon Sep 17 00:00:00 2001 From: Yukihiro Okada Date: Thu, 28 May 2020 02:05:13 +0900 Subject: [PATCH 5/7] Upgrade aws-java-sdk-s3 to 1.11.x --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index b4d8e6a..dca4e92 100644 --- a/build.gradle +++ b/build.gradle @@ -39,7 +39,7 @@ dependencies { compile "org.apache.hadoop:hadoop-hdfs:2.7.5" compile 'org.embulk.input.s3:embulk-util-aws-credentials:0.2.8' - compile "com.amazonaws:aws-java-sdk-s3:1.10.33" + compile "com.amazonaws:aws-java-sdk-s3:1.11.790" compile "org.apache.hadoop:hadoop-aws:2.7.5" compile 'com.google.guava:guava:24.1-jre' From f93bcc8813a05904cb85bea17048b0c4996703ed Mon Sep 17 00:00:00 2001 From: Yukihiro Okada Date: Thu, 28 May 2020 02:18:32 +0900 Subject: [PATCH 6/7] Upgrade guava to 29.0-jre --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index dca4e92..aa9e555 100644 --- a/build.gradle +++ b/build.gradle @@ -41,7 +41,7 @@ dependencies { compile 'org.embulk.input.s3:embulk-util-aws-credentials:0.2.8' compile "com.amazonaws:aws-java-sdk-s3:1.11.790" compile "org.apache.hadoop:hadoop-aws:2.7.5" - compile 'com.google.guava:guava:24.1-jre' + compile 'com.google.guava:guava:29.0-jre' testCompile 'org.jmockit:jmockit:1.38' // testCompile "junit:junit:4.+" From 02167d33229309170406c83df69cef800210a0ed Mon Sep 17 00:00:00 2001 From: Yukihiro Okada Date: Thu, 28 May 2020 02:18:52 +0900 Subject: [PATCH 7/7] Works fine without aws s3 test