Skip to content

change to new mirror, dry kafka version, better curl flags #198

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 3 commits into
base: main
Choose a base branch
from
Open

change to new mirror, dry kafka version, better curl flags #198

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
31f2e73
dry location of kafka and confluent versions
jsvd Jul 3, 2025
f5848d0
use better curl options to still see errors but not the progress meter
jsvd Jul 3, 2025
1d612ed
change mirror to dlcdn.apache.org
jsvd Jul 3, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion .ci/run.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,8 @@ env

set -ex

export KAFKA_VERSION=3.3.1
# Extract apacheKafkaVersion from gradle.properties
export KAFKA_VERSION=$(grep "^apacheKafkaVersion" gradle.properties | cut -d'=' -f2)
./kafka_test_setup.sh

bundle exec rspec -fd
Expand Down
7 changes: 3 additions & 4 deletions build.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,10 +37,9 @@ java {
sourceCompatibility = JavaVersion.VERSION_1_8
}

// given https://docs.confluent.io/current/installation/versions-interoperability.html matrix
// Confluent Platform 7.9.x is Apache Kafka 3.9.x
String confluentKafkaVersion = '7.9.1'
String apacheKafkaVersion = '3.9.1'
// Versions are defined in gradle.properties
String confluentKafkaVersion = project.property('confluentKafkaVersion')
String apacheKafkaVersion = project.property('apacheKafkaVersion')

repositories {
mavenCentral()
Expand Down
6 changes: 6 additions & 0 deletions gradle.properties
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1 +1,7 @@
org.gradle.daemon=false

# given https://docs.confluent.io/current/installation/versions-interoperability.html matrix
# Confluent Platform 7.9.x is Apache Kafka 3.9.x
# Kafka version used by both build.gradle and CI scripts
apacheKafkaVersion=3.9.1
confluentKafkaVersion=7.9.1
7 changes: 4 additions & 3 deletions kafka_test_setup.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,14 +10,15 @@ else
fi

export _JAVA_OPTIONS="-Djava.net.preferIPv4Stack=true"
export CURL_OPTS="--fail --show-error --no-progress-meter --location --retry 3 --retry-delay 5"

rm -rf build
mkdir build

echo "Setup Kafka version $KAFKA_VERSION"
if [ ! -e "kafka_2.13-$KAFKA_VERSION.tgz" ]; then
echo "Kafka not present locally, downloading"
curl -s -o "kafka_2.13-$KAFKA_VERSION.tgz" "https://archive.apache.org/dist/kafka/$KAFKA_VERSION/kafka_2.13-$KAFKA_VERSION.tgz"
curl $CURL_OPTS -o "kafka_2.13-$KAFKA_VERSION.tgz" "https://dlcdn.apache.org/kafka/$KAFKA_VERSION/kafka_2.13-$KAFKA_VERSION.tgz"
fi
cp kafka_2.13-$KAFKA_VERSION.tgz build/kafka.tgz
mkdir build/kafka && tar xzf build/kafka.tgz -C build/kafka --strip-components 1
Expand All @@ -40,7 +41,7 @@ if [ ! -e confluent-community-$CONFLUENT_VERSION.tar.gz ]; then
echo "Confluent Platform not present locally, downloading"
CONFLUENT_MINOR=$(echo "$CONFLUENT_VERSION" | sed -n 's/^\([[:digit:]]*\.[[:digit:]]*\)\.[[:digit:]]*$/\1/p')
echo "CONFLUENT_MINOR is $CONFLUENT_MINOR"
curl -s -o confluent-community-$CONFLUENT_VERSION.tar.gz http://packages.confluent.io/archive/$CONFLUENT_MINOR/confluent-community-$CONFLUENT_VERSION.tar.gz
curl $CURL_OPTS -o confluent-community-$CONFLUENT_VERSION.tar.gz http://packages.confluent.io/archive/$CONFLUENT_MINOR/confluent-community-$CONFLUENT_VERSION.tar.gz
fi
cp confluent-community-$CONFLUENT_VERSION.tar.gz build/confluent_platform.tar.gz
mkdir build/confluent_platform && tar xzf build/confluent_platform.tar.gz -C build/confluent_platform --strip-components 1
Expand Down Expand Up @@ -81,7 +82,7 @@ build/kafka/bin/kafka-topics.sh --create --partitions 1 --replication-factor 1 -
build/kafka/bin/kafka-topics.sh --create --partitions 1 --replication-factor 1 --topic logstash_integration_zstd_topic --bootstrap-server localhost:9092
build/kafka/bin/kafka-topics.sh --create --partitions 3 --replication-factor 1 --topic logstash_integration_partitioner_topic --bootstrap-server localhost:9092
build/kafka/bin/kafka-topics.sh --create --partitions 3 --replication-factor 1 --topic logstash_integration_static_membership_topic --bootstrap-server localhost:9092
curl -s -o build/apache_logs.txt https://s3.amazonaws.com/data.elasticsearch.org/apache_logs/apache_logs.txt
curl $CURL_OPTS -o build/apache_logs.txt https://s3.amazonaws.com/data.elasticsearch.org/apache_logs/apache_logs.txt
cat build/apache_logs.txt | build/kafka/bin/kafka-console-producer.sh --topic logstash_integration_topic_plain --broker-list localhost:9092
cat build/apache_logs.txt | build/kafka/bin/kafka-console-producer.sh --topic logstash_integration_topic_snappy --broker-list localhost:9092 --compression-codec snappy
cat build/apache_logs.txt | build/kafka/bin/kafka-console-producer.sh --topic logstash_integration_topic_lz4 --broker-list localhost:9092 --compression-codec lz4
Expand Down