Skip to content

Commit 60ed028

Browse files
agrgragrgr
authored
FMWK-831 Run all tests nightly (#929)
1 parent 4dbe079 commit 60ed028

File tree

152 files changed

+6809
-198
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

152 files changed

+6809
-198
lines changed

‎.github/workflows/nightly-tests.yml‎

Lines changed: 175 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,175 @@
1+
name: Nightly Tests on GCP
2+
permissions:
3+
contents: read
4+
actions: write # Required to upload artifacts
5+
6+
on:
7+
schedule:
8+
- cron: '0 0 * * *' # Run at midnight UTC
9+
workflow_dispatch: # Allow manual triggering
10+
11+
jobs:
12+
check_changes:
13+
runs-on: ubuntu-latest
14+
timeout-minutes: 5 # Timeout to check for the changes
15+
outputs:
16+
should_run: ${{ steps.check.outputs.changed }}
17+
18+
steps:
19+
- uses: actions/checkout@v5
20+
with:
21+
fetch-depth: 0 # Fetch all history to compare commits
22+
23+
- name: Check for changes since last successful run
24+
id: check
25+
run: |
26+
# Get the last successful run SHA with error handling
27+
echo "Fetching last successful run information..."
28+
29+
# Make API call with error checking
30+
API_RESPONSE=$(curl -s -f -H "Authorization: token ${{ github.token }}" \
31+
"https://api.github.com/repos/${{ github.repository }}/actions/workflows/nightly-tests.yml/runs?status=success&branch=main" || echo "ERROR")
32+
33+
if [ "$API_RESPONSE" = "ERROR" ]; then
34+
echo "API call failed, running tests as fallback"
35+
echo "changed=true" >> $GITHUB_OUTPUT
36+
exit 0
37+
fi
38+
39+
# Try to extract SHA with error handling
40+
LAST_SUCCESS_SHA=$(echo "$API_RESPONSE" | jq -r '.workflow_runs[0].head_sha // ""')
41+
echo "Last successful run SHA: ${LAST_SUCCESS_SHA:-none}"
42+
43+
# Run tests if API returned no results or invalid data
44+
if [ -z "$LAST_SUCCESS_SHA" ] || [ "$LAST_SUCCESS_SHA" = "null" ]; then
45+
echo "No previous successful run found, running tests"
46+
echo "changed=true" >> $GITHUB_OUTPUT
47+
else
48+
# Verify SHA is valid
49+
if ! git rev-parse --quiet --verify "$LAST_SUCCESS_SHA^{commit}" >/dev/null; then
50+
echo "Retrieved SHA is not valid, running tests"
51+
echo "changed=true" >> $GITHUB_OUTPUT
52+
else
53+
# Check if there are any new commits since the last successful run
54+
DIFF=$(git log --oneline $LAST_SUCCESS_SHA..HEAD)
55+
if [ -n "$DIFF" ]; then
56+
echo "Changes detected since last successful run"
57+
echo "changed=true" >> $GITHUB_OUTPUT
58+
else
59+
echo "No changes detected since last successful run"
60+
echo "changed=false" >> $GITHUB_OUTPUT
61+
fi
62+
fi
63+
fi
64+
65+
test:
66+
needs: check_changes
67+
if: ${{ needs.check_changes.outputs.should_run == 'true' || github.event_name == 'workflow_dispatch' }}
68+
runs-on: ubuntu-latest
69+
timeout-minutes: 60 # Timeout for the entire job
70+
env:
71+
TEST_FOLDERS_TO_KEEP: 20 # Number of test result folders to keep
72+
73+
steps:
74+
- name: Checkout for workflow files only
75+
uses: actions/checkout@v4
76+
77+
- name: Authenticate to Google Cloud
78+
uses: google-github-actions/auth@v1
79+
with:
80+
credentials_json: ${{ secrets.GCP_SA_KEY }}
81+
82+
- name: Set up Cloud SDK
83+
uses: google-github-actions/setup-gcloud@v1
84+
85+
- name: Start GCP VM # Start VM
86+
run: |
87+
gcloud compute instances start ${{ secrets.GCP_VM_NAME }} --zone=${{ secrets.GCP_VM_ZONE }}
88+
# Wait for VM to fully boot
89+
sleep 30
90+
91+
- name: Set up VM prerequisites
92+
run: |
93+
# Install Git, JDK 17 and Maven on the VM if not already installed
94+
gcloud compute ssh ${{ secrets.GCP_VM_NAME }} --zone=${{ secrets.GCP_VM_ZONE }} --command="
95+
if ! command -v git &> /dev/null; then
96+
echo 'Installing Git...'
97+
sudo apt-get update
98+
sudo apt-get install -y git
99+
else
100+
echo 'Git is already installed'
101+
fi
102+
103+
# Check for Java
104+
if ! command -v java &> /dev/null || ! java -version 2>&1 | grep -q 'version \"17'; then
105+
echo 'Installing JDK 17...'
106+
sudo apt-get update
107+
sudo apt-get install -y openjdk-17-jdk
108+
else
109+
echo 'JDK 17 is already installed'
110+
fi
111+
112+
# Check for Maven
113+
if ! command -v mvn &> /dev/null; then
114+
echo 'Installing Maven...'
115+
sudo apt-get install -y maven
116+
else
117+
echo 'Maven is already installed'
118+
fi
119+
"
120+
121+
- name: Run tests on VM
122+
id: run_tests
123+
timeout-minutes: 45 # Timeout for just the tests run
124+
run: |
125+
# Get current timestamp for results directory
126+
TIMESTAMP=$(date +"%Y%m%d_%H%M%S")
127+
128+
# Create timestamped directory for this test run
129+
mkdir -p test-results/$TIMESTAMP
130+
131+
# Run commands on the VM to clone repo, set up, and execute tests
132+
gcloud compute ssh ${{ secrets.GCP_VM_NAME }} --zone=${{ secrets.GCP_VM_ZONE }} --command="
133+
# Clean up any previous runs
134+
rm -rf ~/spring_data_repo && mkdir -p ~/spring_data_repo
135+
136+
# Clone the repository
137+
git clone https://github.com/${{ github.repository }} ~/spring_data_repo
138+
cd ~/spring_data_repo
139+
git checkout ${{ github.ref_name }}
140+
141+
# Run tests
142+
mvn clean test -Pall-tests -B -U
143+
144+
# Save exit code to report success/failure
145+
echo \$? > ~/test_exit_code
146+
"
147+
148+
# Copy test results back
149+
gcloud compute scp --recurse ${{ secrets.GCP_VM_NAME }}:~/spring_data_repo/target/surefire-reports ./test-results/$TIMESTAMP --zone=${{ secrets.GCP_VM_ZONE }} || true
150+
151+
# Implement file rotation - keep only N latest test result directories
152+
echo "Rotating test result directories, keeping only the $TEST_FOLDERS_TO_KEEP latest"
153+
ls -t test-results | tail -n +$((TEST_FOLDERS_TO_KEEP + 1)) | xargs -I {} rm -rf test-results/{}
154+
echo "Current test result directories after rotation:"
155+
ls -la test-results/
156+
157+
# Check if tests failed
158+
TEST_EXIT_CODE=$(gcloud compute ssh ${{ secrets.GCP_VM_NAME }} --zone=${{ secrets.GCP_VM_ZONE }} --command="cat ~/test_exit_code || echo 1")
159+
if [ "$TEST_EXIT_CODE" != "0" ]; then
160+
echo "Tests failed with exit code $TEST_EXIT_CODE"
161+
exit 1
162+
fi
163+
164+
- name: Upload test results
165+
uses: actions/upload-artifact@v4
166+
if: always()
167+
with:
168+
name: test-results-${{ github.run_id }}
169+
path: test-results
170+
retention-days: 14 # Days to keep artifact
171+
172+
- name: Stop GCP VM
173+
if: always() # Ensure VM is stopped even if tests fail
174+
run: |
175+
gcloud compute instances stop ${{ secrets.GCP_VM_NAME }} --zone=${{ secrets.GCP_VM_ZONE }}

‎pom.xml‎

Lines changed: 25 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -16,25 +16,25 @@
1616
<parent>
1717
<groupId>org.springframework.data.build</groupId>
1818
<artifactId>spring-data-parent</artifactId>
19-
<version>3.5.3</version>
19+
<version>3.5.4</version>
2020
</parent>
2121

2222
<properties>
2323
<java.version>17</java.version>
2424
<maven.compiler.source>${java.version}</maven.compiler.source>
2525
<maven.compiler.target>${java.version}</maven.compiler.target>
2626
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
27-
<springdata.commons>3.5.3</springdata.commons>
27+
<springdata.commons>3.5.4</springdata.commons>
2828
<spring-boot-starter-test>3.5.3</spring-boot-starter-test>
2929
<spring-cloud-starter-bootstrap>4.3.0</spring-cloud-starter-bootstrap>
30-
<spring-tx>6.2.10</spring-tx>
30+
<spring-tx>6.2.11</spring-tx>
3131
<maven.javadoc.plugin>3.3.0</maven.javadoc.plugin>
3232
<maven.gpg.plugin>1.6</maven.gpg.plugin>
3333
<central.publishing.maven.plugin>0.8.0</central.publishing.maven.plugin>
3434
<aerospike-client-jdk8>9.0.5</aerospike-client-jdk8>
3535
<aerospike-reactor-client>9.0.5</aerospike-reactor-client>
3636
<reactor-test>3.7.7</reactor-test>
37-
<embedded-aerospike>3.1.14</embedded-aerospike>
37+
<embedded-aerospike>3.1.16</embedded-aerospike>
3838
<jodatime>2.14.0</jodatime>
3939
<lombok>1.18.38</lombok>
4040
<awaitility>4.3.0</awaitility>
@@ -157,6 +157,26 @@
157157
</developer>
158158
</developers>
159159

160+
<profiles>
161+
<!-- Default profile with extensive tests excluded, active by default -->
162+
<profile>
163+
<id>default</id>
164+
<activation>
165+
<activeByDefault>true</activeByDefault>
166+
</activation>
167+
<properties>
168+
<excludedGroups>extensive</excludedGroups>
169+
</properties>
170+
</profile>
171+
<!-- All tests -->
172+
<profile>
173+
<id>all-tests</id>
174+
<properties>
175+
<excludedGroups/>
176+
</properties>
177+
</profile>
178+
</profiles>
179+
160180
<scm>
161181
<connection>scm:git:git://github.com/spring-projects/spring-data-aerospike.git</connection>
162182
<developerConnection>scm:git:ssh://[email protected]:aerospike/spring-data-aerospike.git</developerConnection>
@@ -396,6 +416,7 @@
396416
<include>**/*Test.java</include>
397417
<include>**/*Tests.java</include>
398418
</includes>
419+
<excludedGroups>${excludedGroups}</excludedGroups>
399420
<argLine>
400421
--add-opens java.base/java.util=ALL-UNNAMED
401422
--add-opens java.base/java.net=ALL-UNNAMED

‎src/test/java/org/springframework/data/aerospike/BaseBlockingIntegrationTests.java‎

Lines changed: 56 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
package org.springframework.data.aerospike;
22

33
import com.aerospike.client.IAerospikeClient;
4+
import com.aerospike.client.exp.Expression;
5+
import com.aerospike.client.query.Filter;
46
import com.aerospike.client.query.Statement;
57
import org.junit.jupiter.api.TestInfo;
68
import org.springframework.beans.factory.annotation.Autowired;
@@ -10,7 +12,6 @@
1012
import org.springframework.data.aerospike.config.BlockingTestConfig;
1113
import org.springframework.data.aerospike.config.CommonTestConfig;
1214
import org.springframework.data.aerospike.config.IndexedBinsAnnotationsProcessor;
13-
import org.springframework.data.aerospike.convert.AerospikeConverter;
1415
import org.springframework.data.aerospike.core.AerospikeTemplate;
1516
import org.springframework.data.aerospike.mapping.AerospikePersistentProperty;
1617
import org.springframework.data.aerospike.mapping.BasicAerospikePersistentEntity;
@@ -167,6 +168,60 @@ protected boolean queryHasSecIndexFilter(String namespace, String setName, Query
167168
return statement.getFilter() != null;
168169
}
169170

171+
/**
172+
* Assert that the given method's query statement does not contain secondary index filter
173+
*
174+
* @param methodName Query method to be performed
175+
* @param returnEntityClass Class of Query return entity
176+
* @param methodParams Query parameters
177+
*/
178+
protected void assertQueryHasNoSecIndexFilter(String methodName, Class<?> returnEntityClass,
179+
Object... methodParams) {
180+
assertThat(queryHasSecIndexFilter(methodName, returnEntityClass, methodParams)).isFalse();
181+
}
182+
183+
protected Filter getQuerySecIndexFilter(String methodName, Class<?> returnEntityClass, Object... methodParams) {
184+
String setName = template.getSetName(returnEntityClass);
185+
String[] binNames = getBinNamesFromTargetClass(returnEntityClass, mappingContext);
186+
Query query = QueryUtils.createQueryForMethodWithArgs(serverVersionSupport, methodName, methodParams);
187+
188+
return getQuerySecIndexFilter(namespace, setName, query, binNames);
189+
}
190+
191+
protected Filter getQuerySecIndexFilter(Query query, Class<?> returnEntityClass) {
192+
String setName = template.getSetName(returnEntityClass);
193+
String[] binNames = getBinNamesFromTargetClass(returnEntityClass, mappingContext);
194+
195+
return getQuerySecIndexFilter(namespace, setName, query, binNames);
196+
}
197+
198+
protected Filter getQuerySecIndexFilter(String namespace, String setName, Query query, String[] binNames) {
199+
QueryContext queryContext = queryEngine.getQueryContextBuilder().build(namespace, setName, query, binNames);
200+
// Checking that the statement has secondary index filter (which means it will be used)
201+
return queryContext.statement().getFilter();
202+
}
203+
204+
protected Expression getQueryExpression(String methodName, Class<?> returnEntityClass, Object... methodParams) {
205+
String setName = template.getSetName(returnEntityClass);
206+
String[] binNames = getBinNamesFromTargetClass(returnEntityClass, mappingContext);
207+
Query query = QueryUtils.createQueryForMethodWithArgs(serverVersionSupport, methodName, methodParams);
208+
209+
return getQueryExpression(namespace, setName, query, binNames);
210+
}
211+
212+
protected Expression getQueryExpression(Query query, Class<?> returnEntityClass) {
213+
String setName = template.getSetName(returnEntityClass);
214+
String[] binNames = getBinNamesFromTargetClass(returnEntityClass, mappingContext);
215+
216+
return getQueryExpression(namespace, setName, query, binNames);
217+
}
218+
219+
protected Expression getQueryExpression(String namespace, String setName, Query query, String[] binNames) {
220+
QueryContext queryContext = queryEngine.getQueryContextBuilder().build(namespace, setName, query, binNames);
221+
// Checking that the statement has secondary index filter (which means it will be used)
222+
return queryEngine.getFilterExpressionsBuilder().build(queryContext.qualifier());
223+
}
224+
170225
protected Map<?, ?> pojoToMap(Object pojo) {
171226
Object result = template.getAerospikeConverter().toWritableValue(pojo, TypeInformation.of(pojo.getClass()));
172227
if (result instanceof Map<?, ?>) {

‎src/test/java/org/springframework/data/aerospike/BaseIntegrationTests.java‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,9 +4,15 @@
44
import org.junit.jupiter.api.BeforeEach;
55
import org.springframework.beans.factory.annotation.Autowired;
66
import org.springframework.beans.factory.annotation.Value;
7+
import org.springframework.data.aerospike.query.model.Index;
78
import org.springframework.data.aerospike.util.AdditionalAerospikeTestOperations;
89
import org.springframework.data.aerospike.util.AerospikeUniqueId;
910

11+
import java.util.Comparator;
12+
import java.util.Set;
13+
import java.util.stream.Collectors;
14+
import java.util.stream.Stream;
15+
1016
public abstract class BaseIntegrationTests {
1117

1218
public static final String DEFAULT_SET_NAME = "aerospike";
@@ -39,4 +45,25 @@ public void setUp() {
3945
protected String getNameSpace() {
4046
return namespace;
4147
}
48+
49+
/**
50+
* Returns bin name chosen for secondary index Filter based either on cardinality of indexes
51+
* or on sequential order of supplied indexes (returns the first one if cardinality is not the same for all).
52+
*
53+
* @param indexes One or more indexes to extract cardinality and bin names from
54+
* @return Bin name if indexes are not empty, otherwise null
55+
*/
56+
public static String getBinNameForFilter(Index... indexes) {
57+
if (indexes == null || indexes.length == 0) return null;
58+
Set<Integer> distinctCardinalityValues = Stream.of(indexes)
59+
.map(Index::getBinValuesRatio)
60+
.collect(Collectors.toSet());
61+
if (distinctCardinalityValues.size() > 1) {
62+
return Stream.of(indexes)
63+
.min(Comparator.comparing(Index::getBinValuesRatio))
64+
.get()
65+
.getBin();
66+
}
67+
return indexes[0].getBin();
68+
}
4269
}

0 commit comments

Comments
 (0)