Skip to content

New GenerateMetadataTask gradle task for collecting metadata for specific library version #686

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

New GenerateMetadataTask gradle task for collecting metadata for specific library version #686

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions tests/tck-build-logic/src/main/groovy/org.graalvm.internal.tck-harness.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ import org.graalvm.internal.tck.DockerTask
import org.graalvm.internal.tck.ConfigFilesChecker
import org.graalvm.internal.tck.ScaffoldTask
import org.graalvm.internal.tck.GrypeTask
import org.graalvm.internal.tck.GenerateMetadataTask
import org.graalvm.internal.tck.TestedVersionUpdaterTask
import org.graalvm.internal.tck.harness.tasks.TestInvocationTask
import org.graalvm.internal.tck.harness.tasks.CheckstyleInvocationTask
Expand Down Expand Up @@ -217,6 +218,11 @@ tasks.register("contribute", ContributionTask.class) { task ->
task.setGroup(METADATA_GROUP)
}

tasks.register("generateMetadata", GenerateMetadataTask.class) { task ->
task.setDescription("Generates metadata and prepares pull request for contibuting on metadata repository based on provided tests.")
task.setGroup(METADATA_GROUP)
}

tasks.register("checkConfigFiles", ConfigFilesChecker.class) { task ->
task.setDescription("Checks content of config files for a new library.")
task.setGroup(METADATA_GROUP)
Expand Down
178 changes: 40 additions & 138 deletions tests/tck-build-logic/src/main/java/org/graalvm/internal/tck/ContributionTask.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,6 @@

import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.core.util.DefaultIndenter;
import com.fasterxml.jackson.core.util.DefaultPrettyPrinter;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.SerializationFeature;
import org.graalvm.internal.tck.exceptions.ContributingException;
Expand All @@ -12,23 +10,20 @@
import org.graalvm.internal.tck.utils.ConfigurationStringBuilder;
import org.graalvm.internal.tck.utils.FilesUtils;
import org.graalvm.internal.tck.utils.InteractiveTaskUtils;
import org.graalvm.internal.tck.utils.MetadataGenerationUtils;
import org.gradle.api.DefaultTask;
import org.gradle.api.file.FileSystemOperations;
import org.gradle.api.file.ProjectLayout;
import org.gradle.api.tasks.TaskAction;
import org.gradle.process.ExecOperations;

import javax.inject.Inject;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
Expand All @@ -40,8 +35,7 @@
public abstract class ContributionTask extends DefaultTask {
private static final String BRANCH_NAME_PREFIX = "add-support-for-";
private static final String METADATA_INDEX = "metadata/index.json";
private static final String BUILD_FILE = "build.gradle";
private static final String USER_CODE_FILTER_FILE = "user-code-filter.json";
private static final String GRADLEW = "gradlew";
private static final String REQUIRED_DOCKER_IMAGES_FILE = "required-docker-images.txt";

@Inject
Expand All @@ -55,24 +49,15 @@ public abstract class ContributionTask extends DefaultTask {

private final ObjectMapper objectMapper = new ObjectMapper().enable(SerializationFeature.INDENT_OUTPUT).setSerializationInclusion(JsonInclude.Include.NON_NULL);

private Path gradlew;
private Path gradlewPath;
private Path testsDirectory;
private Path metadataDirectory;

private Coordinates coordinates;
String coordinates;

private record ContributingQuestion(String question, String help) {}
private final Map<String, ContributingQuestion> questions = new HashMap<>();

private void initializeWorkingDirectories(){
testsDirectory = getPathFromProject(CoordinateUtils.replace("tests/src/$group$/$artifact$/$version$", coordinates));
metadataDirectory = getPathFromProject(CoordinateUtils.replace("metadata/$group$/$artifact$/$version$", coordinates));
gradlew = getPathFromProject("gradlew");
private record ContributingQuestion(String question, String help) {
}

private Path getPathFromProject(String fileName) {
return Path.of(getProjectFile(fileName).getAbsolutePath());
}
private final Map<String, ContributingQuestion> questions = new HashMap<>();

private File getProjectFile(String fileName) {
return getLayout().getProjectDirectory().file(fileName).getAsFile();
Expand All @@ -96,7 +81,7 @@ void run() throws IOException {
coordinates = getCoordinates();
InteractiveTaskUtils.closeSection();

Path coordinatesMetadataRoot = getPathFromProject(CoordinateUtils.replace("metadata/$group$/$artifact$", coordinates));
Path coordinatesMetadataRoot = MetadataGenerationUtils.getPathFromProject(getLayout(), CoordinateUtils.replace("metadata/$group$/$artifact$", CoordinateUtils.fromString(coordinates)));
boolean isExistingLibrary = Files.exists(coordinatesMetadataRoot);

Path testsLocation = getTestsLocation();
Expand All @@ -111,29 +96,31 @@ void run() throws IOException {
List<String> packages = getAllowedPackages();
InteractiveTaskUtils.closeSection();

List<Coordinates> additionalTestImplementationDependencies = getAdditionalDependencies();
List<String> additionalTestImplementationDependencies = getAdditionalDependencies();
InteractiveTaskUtils.closeSection();

// initialize project
initializeWorkingDirectories();
gradlewPath = MetadataGenerationUtils.getPathFromProject(getLayout(), GRADLEW);
testsDirectory = MetadataGenerationUtils.computeTestsDirectory(getLayout(), coordinates);

createStubs(isExistingLibrary);
updateAllowedPackages(packages, isExistingLibrary);

// generate necessary boilerplate code
addTests(testsLocation);
addResources(resourcesLocation);
addDockerImages(dockerImages);
addUserCodeFilterFile(packages);
MetadataGenerationUtils.addUserCodeFilterFile(testsDirectory, packages);
addAdditionalDependencies(additionalTestImplementationDependencies);
addAgentConfigBlock();
MetadataGenerationUtils.addAgentConfigBlock(testsDirectory);

// run agent in conditional mode
collectMetadata();
MetadataGenerationUtils.collectMetadata(getExecOperations(), testsDirectory, getLayout(), coordinates, gradlewPath);

// create a PR
boolean shouldCreatePR = shouldCreatePullRequest();
if (shouldCreatePR) {
String branch = BRANCH_NAME_PREFIX + coordinates.toString().replace(':', '-');
String branch = BRANCH_NAME_PREFIX + coordinates.replace(':', '-');
createPullRequest(branch);

InteractiveTaskUtils.printUserInfo("After your pull requests gets generated, please update the pull request description to mention all places where your pull request" +
Expand All @@ -143,15 +130,9 @@ void run() throws IOException {
InteractiveTaskUtils.printSuccessfulStatement("Contribution successfully completed! Thank you!");
}

private Coordinates getCoordinates() {
private String getCoordinates() {
ContributingQuestion question = questions.get("coordinates");
return InteractiveTaskUtils.askQuestion(question.question(), question.help(), (answer) -> {
try {
return CoordinateUtils.fromString(answer);
} catch (IllegalArgumentException ex) {
throw new ContributingException(ex.getMessage());
}
});
return InteractiveTaskUtils.askQuestion(question.question(), question.help(), (answer) -> answer);
}

private Path getTestsLocation() {
Expand Down Expand Up @@ -200,7 +181,7 @@ private void checkPackages(Path testsPath) throws ContributingException {
}
}

private Path getResourcesLocation(){
private Path getResourcesLocation() {
ContributingQuestion question = questions.get("resourcesLocation");
return InteractiveTaskUtils.askQuestion(question.question(), question.help(), (answer) -> {
if (answer.equalsIgnoreCase("-")) {
Expand Down Expand Up @@ -241,21 +222,15 @@ private List<String> getAllowedPackages() {
return InteractiveTaskUtils.askRecurringQuestions(question.question(), question.help(), 1, answer -> answer);
}

private List<Coordinates> getAdditionalDependencies() {
private List<String> getAdditionalDependencies() {
ContributingQuestion question = questions.get("additionalDependencies");
return InteractiveTaskUtils.askRecurringQuestions(question.question(), question.help(), 0, answer -> {
try {
return CoordinateUtils.fromString(answer);
} catch (IllegalArgumentException ex) {
throw new ContributingException(ex.getMessage());
}
});
return InteractiveTaskUtils.askRecurringQuestions(question.question(), question.help(), 0, answer -> answer);
}

private void createStubs(boolean shouldUpdate) {
InteractiveTaskUtils.printUserInfo("Generating stubs for: " + coordinates );
InteractiveTaskUtils.printUserInfo("Generating stubs for: " + coordinates);
String opt = shouldUpdate ? "--update" : "";
invokeCommand(gradlew + " scaffold --coordinates " + coordinates + " --skipTests " + opt, "Cannot generate stubs for: " + coordinates);
MetadataGenerationUtils.invokeCommand(getExecOperations(), gradlewPath + " scaffold --coordinates " + coordinates + " --skipTests " + opt, "Cannot generate stubs for: " + coordinates);
}

private void updateAllowedPackages(List<String> allowedPackages, boolean libraryAlreadyExists) throws IOException {
Expand All @@ -264,8 +239,9 @@ private void updateAllowedPackages(List<String> allowedPackages, boolean library

List<MetadataIndexEntry> entries = objectMapper.readValue(metadataIndex, new TypeReference<>() {});
int replaceEntryIndex = -1;
Coordinates dependencyCoordinates = CoordinateUtils.fromString(coordinates);
for (int i = 0; i < entries.size(); i++) {
if (entries.get(i).module().equals(coordinates.group() + ":" + coordinates.artifact())) {
if (entries.get(i).module().equals(dependencyCoordinates.group() + ":" + dependencyCoordinates.artifact())) {
replaceEntryIndex = i;
break;
}
Expand All @@ -289,7 +265,7 @@ private void updateAllowedPackages(List<String> allowedPackages, boolean library
objectMapper.writeValue(metadataIndex, entries);
}

private void addTests(Path originalTestsLocation){
private void addTests(Path originalTestsLocation) {
Path destination = testsDirectory.resolve("src").resolve("test").resolve("java");
Path allTests = originalTestsLocation.resolve(".");

Expand All @@ -301,7 +277,7 @@ private void addTests(Path originalTestsLocation){
});
}

private void addResources(Path originalResourcesDirectory){
private void addResources(Path originalResourcesDirectory) {
if (originalResourcesDirectory == null) {
return;
}
Expand All @@ -311,8 +287,8 @@ private void addResources(Path originalResourcesDirectory){

InteractiveTaskUtils.printUserInfo("Copying resources from: " + originalResourcesDirectory + " to " + destination);
getFileSystemOperations().copy(copySpec -> {
copySpec.from(originalResourcesDirectory);
copySpec.into(destination);
copySpec.from(originalResourcesDirectory);
copySpec.into(destination);
});
}

Expand All @@ -326,39 +302,24 @@ private void addDockerImages(List<String> images) throws IOException {
ensureFileBelongsToProject(destination);

if (!Files.exists(destination)) {
Files.createFile(destination);
Files.createDirectories(destination.getParent());
}

for (String image : images) {
writeToFile(destination, image.concat(System.lineSeparator()), StandardOpenOption.APPEND);
MetadataGenerationUtils.writeToFile(destination, image.concat(System.lineSeparator()), StandardOpenOption.APPEND);
}
}

private void addUserCodeFilterFile(List<String> packages) throws IOException {
InteractiveTaskUtils.printUserInfo("Generating " + USER_CODE_FILTER_FILE);
List<Map<String, String>> filterFileRules = new ArrayList<>();

// add exclude classes
filterFileRules.add(Map.of("excludeClasses", "**"));

// add include classes
packages.forEach(p -> filterFileRules.add(Map.of("includeClasses", p + ".**")));

DefaultPrettyPrinter prettyPrinter = new DefaultPrettyPrinter();
prettyPrinter.indentArraysWith(DefaultIndenter.SYSTEM_LINEFEED_INSTANCE);
objectMapper.writer(prettyPrinter).writeValue(testsDirectory.resolve(USER_CODE_FILTER_FILE).toFile(), Map.of("rules", filterFileRules));
}

private void addAdditionalDependencies(List<Coordinates> dependencies) throws IOException {
private void addAdditionalDependencies(List<String> dependencies) throws IOException {
if (dependencies.isEmpty()) {
return;
}

Path buildFilePath = testsDirectory.resolve(BUILD_FILE);
InteractiveTaskUtils.printUserInfo("Adding following dependencies to " + BUILD_FILE + " file: " + dependencies);
Path buildFilePath = testsDirectory.resolve(MetadataGenerationUtils.BUILD_FILE);
InteractiveTaskUtils.printUserInfo("Adding following dependencies to " + MetadataGenerationUtils.BUILD_FILE + " file: " + dependencies);

if (!Files.isRegularFile(buildFilePath)) {
throw new RuntimeException("Cannot add additional dependencies to " + buildFilePath + ". Please check if a " + BUILD_FILE + " exists on that location.");
throw new RuntimeException("Cannot add additional dependencies to " + buildFilePath + ". Please check if a " + MetadataGenerationUtils.BUILD_FILE + " exists on that location.");
}

ConfigurationStringBuilder sb = new ConfigurationStringBuilder();
Expand All @@ -368,39 +329,13 @@ private void addAdditionalDependencies(List<Coordinates> dependencies) throws IO
if (line.trim().equalsIgnoreCase("dependencies {")) {
sb.indent();
for (var dependency : dependencies) {
sb.append("testImplementation").space().quote(dependency.toString()).newLine();
sb.append("testImplementation").space().quote(dependency).newLine();
}
sb.unindent();
}
}

writeToFile(buildFilePath, sb.toString(), StandardOpenOption.WRITE);
}

private void addAgentConfigBlock() throws IOException {
Path buildFilePath = testsDirectory.resolve(BUILD_FILE);
InteractiveTaskUtils.printUserInfo("Configuring agent block in: " + BUILD_FILE);

if (!Files.isRegularFile(buildFilePath)) {
throw new RuntimeException("Cannot add agent block to " + buildFilePath + ". Please check if a " + BUILD_FILE + " exists on that location.");
}

try(InputStream stream = ContributionTask.class.getResourceAsStream("/contributing/agent.template")) {
if (stream == null) {
throw new RuntimeException("Cannot find template for the graalvm configuration block");
}

String content = System.lineSeparator() + (new String(stream.readAllBytes(), StandardCharsets.UTF_8));
writeToFile(buildFilePath, content, StandardOpenOption.APPEND);
}
}

private void collectMetadata() {
InteractiveTaskUtils.printUserInfo("Generating metadata");
invokeCommand(gradlew + " -Pagent test", "Cannot generate metadata", testsDirectory);

InteractiveTaskUtils.printUserInfo("Performing metadata copy");
invokeCommand(gradlew + " metadataCopy --task test --dir " + metadataDirectory, "Cannot perform metadata copy", testsDirectory);
MetadataGenerationUtils.writeToFile(buildFilePath, sb.toString(), StandardOpenOption.WRITE);
}

private boolean shouldCreatePullRequest() {
Expand All @@ -410,53 +345,20 @@ private boolean shouldCreatePullRequest() {

private void createPullRequest(String branch) {
InteractiveTaskUtils.printUserInfo("Creating new branch: " + branch);
invokeCommand("git switch -C " + branch, "Cannot create a new branch");
MetadataGenerationUtils.invokeCommand(getExecOperations(), "git switch -C " + branch, "Cannot create a new branch");

InteractiveTaskUtils.printUserInfo("Staging changes");
invokeCommand("git add .", "Cannot add changes");
MetadataGenerationUtils.invokeCommand(getExecOperations(), "git add .", "Cannot add changes");

InteractiveTaskUtils.printUserInfo("Committing changes");
invokeCommand("git", List.of("commit", "-m", "Add metadata for " + coordinates), "Cannot commit changes", null);
MetadataGenerationUtils.invokeCommand(getExecOperations(), "git", List.of("commit", "-m", "Add metadata for " + coordinates), "Cannot commit changes", null);

InteractiveTaskUtils.printUserInfo("Pushing changes");
invokeCommand("git push origin " + branch, "Cannot push to origin");
MetadataGenerationUtils.invokeCommand(getExecOperations(), "git push origin " + branch, "Cannot push to origin");

InteractiveTaskUtils.printUserInfo("Complete pull request creation on the above link");
}

private void writeToFile(Path path, String content, StandardOpenOption writeOption) throws IOException {
Files.createDirectories(path.getParent());
Files.writeString(path, content, StandardCharsets.UTF_8, writeOption);
}

private void invokeCommand(String command, String errorMessage) {
invokeCommand(command, errorMessage, null);
}

private void invokeCommand(String command, String errorMessage, Path workingDirectory) {
String[] commandParts = command.split(" ");
String executable = commandParts[0];

List<String> args = List.of(Arrays.copyOfRange(commandParts, 1, commandParts.length));
invokeCommand(executable, args, errorMessage, workingDirectory);
}

private void invokeCommand(String executable, List<String> args, String errorMessage, Path workingDirectory) {
ByteArrayOutputStream execOutput = new ByteArrayOutputStream();
var result = getExecOperations().exec(execSpec -> {
if (workingDirectory != null) {
execSpec.setWorkingDir(workingDirectory);
}
execSpec.setExecutable(executable);
execSpec.setArgs(args);
execSpec.setStandardOutput(execOutput);
});

if (result.getExitValue() != 0) {
throw new RuntimeException(errorMessage + ". See: " + execOutput);
}
}

private void ensureFileBelongsToProject(Path file) {
if (!file.isAbsolute() || !file.startsWith(getLayout().getProjectDirectory().getAsFile().getAbsolutePath())) {
throw new RuntimeException("The following file doesn't belong to the metadata repository: " + file);
Expand Down
Loading