feat: Support Imagen 4 Ingredients on Vertex

PiperOrigin-RevId: 808749595

Author: matthew29tang

examples/src/main/java/com/google/genai/examples/Constants.java

@@ -40,6 +40,9 @@ private Constants() {}
   /** The name of the Imagen model to be used for image editing in the examples. */
   public static final String IMAGEN_CAPABILITY_MODEL_NAME = "imagen-3.0-capability-001";
 
+  /** The name of the Imagen ingredients model to be used in the examples. */
+  public static final String IMAGEN_INGREDIENTS_MODEL_NAME = "imagen-4.0-ingredients-preview";
+
   /** The name of the Imagen product recontext model to be used in the examples. */
   public static final String IMAGEN_RECONTEXT_MODEL_NAME = "imagen-product-recontext-preview-06-30";
 

examples/src/main/java/com/google/genai/examples/EditImageContentReference.java

@@ -0,0 +1,114 @@
+/*
+ * Copyright 2025 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Usage:
+ *
+ * <p>1a. If you are using Vertex AI, setup ADC to get credentials:
+ * https://cloud.google.com/docs/authentication/provide-credentials-adc#google-idp
+ *
+ * <p>Then set Project, Location, and USE_VERTEXAI flag as environment variables:
+ *
+ * <p>export GOOGLE_CLOUD_PROJECT=YOUR_PROJECT
+ *
+ * <p>export GOOGLE_CLOUD_LOCATION=YOUR_LOCATION
+ *
+ * <p>export GOOGLE_GENAI_USE_VERTEXAI=true
+ *
+ * <p>1b. If you are using Gemini Developer API, set an API key environment variable. You can find a
+ * list of available API keys here: https://aistudio.google.com/app/apikey
+ *
+ * <p>export GOOGLE_API_KEY=YOUR_API_KEY
+ *
+ * <p>2. Compile the java package and run the sample code.
+ *
+ * <p>mvn clean compile
+ *
+ * <p>mvn exec:java -Dexec.mainClass="com.google.genai.examples.EditImageContentReference"
+ * -Dexec.args="YOUR_MODEL_ID"
+ */
+package com.google.genai.examples;
+
+import com.google.genai.Client;
+import com.google.genai.types.ContentReferenceImage;
+import com.google.genai.types.EditImageConfig;
+import com.google.genai.types.EditImageResponse;
+import com.google.genai.types.Image;
+import com.google.genai.types.ReferenceImage;
+import com.google.genai.types.StyleReferenceConfig;
+import com.google.genai.types.StyleReferenceImage;
+import java.util.ArrayList;
+
+/** An example of using the Unified Gen AI Java SDK to edit an image (Mask reference). */
+public final class EditImageContentReference {
+  public static void main(String[] args) {
+    final String modelId;
+    if (args.length != 0) {
+      modelId = args[0];
+    } else {
+      modelId = Constants.IMAGEN_INGREDIENTS_MODEL_NAME;
+    }
+
+    // Instantiate the client. The client by default uses the Gemini Developer API. It gets the API
+    // key from the environment variable `GOOGLE_API_KEY`. Vertex AI API can be used by setting the
+    // environment variables `GOOGLE_CLOUD_LOCATION` and `GOOGLE_CLOUD_PROJECT`, as well as setting
+    // `GOOGLE_GENAI_USE_VERTEXAI` to "true".
+    //
+    // Note: Some services are only available in a specific API backend (Gemini or Vertex), you will
+    // get a `UnsupportedOperationException` if you try to use a service that is not available in
+    // the backend you are using.
+    Client client = new Client();
+
+    if (client.vertexAI()) {
+      System.out.println("Using Vertex AI");
+    } else {
+      System.out.println("Gemini Developer API is not supported for this example.");
+      System.exit(0);
+    }
+
+    EditImageConfig editImageConfig =
+        EditImageConfig.builder().numberOfImages(1).outputMimeType("image/jpeg").build();
+
+    ArrayList<ReferenceImage> referenceImages = new ArrayList<>();
+    Image dogImage = Image.builder().gcsUri("gs://genai-sdk-tests/inputs/images/dog.jpg").build();
+    ContentReferenceImage contentReferenceImage =
+        ContentReferenceImage.builder().referenceImage(dogImage).referenceId(1).build();
+    referenceImages.add(contentReferenceImage);
+
+    Image cyberpunkImage =
+        Image.builder().gcsUri("gs://genai-sdk-tests/inputs/images/cyberpunk.jpg").build();
+    StyleReferenceImage styleReferenceImage =
+        StyleReferenceImage.builder()
+            .referenceId(2)
+            .referenceImage(cyberpunkImage)
+            .config(StyleReferenceConfig.builder().styleDescription("cyberpunk style").build())
+            .build();
+    referenceImages.add(styleReferenceImage);
+
+    EditImageResponse editImageResponse =
+        client.models.editImage(
+            modelId,
+            "Dog in [1] sleeping on the ground at the bottom of the image with the cyberpunk city"
+                + " landscape in [2] in the background visible on the side of the mug.",
+            referenceImages,
+            editImageConfig);
+
+    Image editedImage = editImageResponse.generatedImages().get().get(0).image().get();
+    // Do something with editedImage.
+  }
+
+  private EditImageContentReference() {}
+}

src/main/java/com/google/genai/Models.java

@@ -7967,12 +7967,13 @@ public GenerateImagesResponse generateImages(
    * @param model the name of the GenAI model to use for editing capabilities
    * @param prompt the prompt to edit the image
    * @param referenceImages a {@link List<com.google.genai.types.ReferenceImage>} to send to use for
-   *     editing. The 5 types of reference images are: {@link
+   *     editing. The 6 types of reference images are: {@link
    *     com.google.genai.types.RawReferenceImage}, {@link
    *     com.google.genai.types.MaskReferenceImage}, {@link
    *     com.google.genai.types.ControlReferenceImage}, {@link
    *     com.google.genai.types.StyleReferenceImage}, {@link
-   *     com.google.genai.types.SubjectReferenceImage},
+   *     com.google.genai.types.SubjectReferenceImage}, {@link
+   *     com.google.genai.types.ContentReferenceImage}
    * @param config a {@link com.google.genai.types.EditImageConfig} instance that specifies the
    *     optional configurations
    * @return a {@link com.google.genai.types.EditImageResponse} instance that contains the edited

src/main/java/com/google/genai/types/ContentReferenceImage.java

@@ -0,0 +1,118 @@
+/*
+ * Copyright 2025 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Auto-generated code. Do not edit.
+
+package com.google.genai.types;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
+import com.google.auto.value.AutoValue;
+import com.google.genai.JsonSerializable;
+import java.util.Optional;
+
+/**
+ * A content reference image.
+ *
+ * <p>A content reference image represents a subject to reference (ex. person, product, animal)
+ * provided by the user. It can optionally be provided in addition to a style reference image (ex.
+ * background, style reference).
+ */
+@AutoValue
+@JsonDeserialize(builder = ContentReferenceImage.Builder.class)
+public abstract class ContentReferenceImage extends JsonSerializable implements ReferenceImage {
+  /** The reference image for the editing operation. */
+  @JsonProperty("referenceImage")
+  public abstract Optional<Image> referenceImage();
+
+  /** The id of the reference image. */
+  @JsonProperty("referenceId")
+  public abstract Optional<Integer> referenceId();
+
+  /** The type of the reference image. Only set by the SDK. */
+  @JsonProperty("referenceType")
+  public abstract Optional<String> referenceType();
+
+  /** Instantiates a builder for ContentReferenceImage. */
+  @ExcludeFromGeneratedCoverageReport
+  public static Builder builder() {
+    return new AutoValue_ContentReferenceImage.Builder();
+  }
+
+  /** Creates a builder with the same values as this instance. */
+  public abstract Builder toBuilder();
+
+  /** Builder for ContentReferenceImage. */
+  @AutoValue.Builder
+  public abstract static class Builder {
+    /** For internal usage. Please use `ContentReferenceImage.builder()` for instantiation. */
+    @JsonCreator
+    private static Builder create() {
+      return new AutoValue_ContentReferenceImage.Builder();
+    }
+
+    /**
+     * Setter for referenceImage.
+     *
+     * <p>referenceImage: The reference image for the editing operation.
+     */
+    @JsonProperty("referenceImage")
+    public abstract Builder referenceImage(Image referenceImage);
+
+    /**
+     * Setter for referenceImage builder.
+     *
+     * <p>referenceImage: The reference image for the editing operation.
+     */
+    public Builder referenceImage(Image.Builder referenceImageBuilder) {
+      return referenceImage(referenceImageBuilder.build());
+    }
+
+    /**
+     * Setter for referenceId.
+     *
+     * <p>referenceId: The id of the reference image.
+     */
+    @JsonProperty("referenceId")
+    public abstract Builder referenceId(Integer referenceId);
+
+    /**
+     * Setter for referenceType.
+     *
+     * <p>referenceType: The type of the reference image. Only set by the SDK.
+     */
+    @JsonProperty("referenceType")
+    public abstract Builder referenceType(String referenceType);
+
+    public abstract ContentReferenceImage build();
+  }
+
+  /** Deserializes a JSON string to a ContentReferenceImage object. */
+  @ExcludeFromGeneratedCoverageReport
+  public static ContentReferenceImage fromJson(String jsonString) {
+    return JsonSerializable.fromJsonString(jsonString, ContentReferenceImage.class);
+  }
+
+  @Override
+  public ReferenceImageAPI toReferenceImageAPI() {
+    ReferenceImageAPI.Builder referenceImageAPIBuilder = ReferenceImageAPI.builder();
+    referenceImage().ifPresent(referenceImageAPIBuilder::referenceImage);
+    referenceId().ifPresent(referenceImageAPIBuilder::referenceId);
+    referenceImageAPIBuilder.referenceType("REFERENCE_TYPE_CONTENT");
+    return referenceImageAPIBuilder.build();
+  }
+}

src/main/java/com/google/genai/types/EditImageParameters.java

@@ -41,7 +41,7 @@ public abstract class EditImageParameters extends JsonSerializable {
   @JsonProperty("prompt")
   public abstract Optional<String> prompt();
 
-  /** The reference images for Imagen 3 editing. */
+  /** The reference images for editing. */
   @JsonProperty("referenceImages")
   public abstract Optional<List<ReferenceImageAPI>> referenceImages();
 
@@ -86,15 +86,15 @@ private static Builder create() {
     /**
      * Setter for referenceImages.
      *
-     * <p>referenceImages: The reference images for Imagen 3 editing.
+     * <p>referenceImages: The reference images for editing.
      */
     @JsonProperty("referenceImages")
     public abstract Builder referenceImages(List<ReferenceImageAPI> referenceImages);
 
     /**
      * Setter for referenceImages.
      *
-     * <p>referenceImages: The reference images for Imagen 3 editing.
+     * <p>referenceImages: The reference images for editing.
      */
     public Builder referenceImages(ReferenceImageAPI... referenceImages) {
       return referenceImages(Arrays.asList(referenceImages));