diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..de1ef77 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +node_modules/ +.gradle/ +build/ +dist/ +bin/ \ No newline at end of file diff --git a/tools/README.md b/tools/README.md index f877092..ea60a0b 100644 --- a/tools/README.md +++ b/tools/README.md @@ -1,4 +1,7 @@ # Intelligent Advisor tools +## project-history-migrator-tool +A CLI utility that allows the migration of the full project version history from a self-managed instance of Intelligent Advisor to a cloud instance. The tool migrates the project version history for both Policy Modeling projects and Decision Service projects. + ## web-projects-migrator A tool for migrating rules in Oracle Policy Modeling projects to Intelligent Advisor Flow projects and Decision Service Projects. Supplied in source form to encourage project-specific enhancements. diff --git a/tools/project-history-migrator-tool/README.md b/tools/project-history-migrator-tool/README.md new file mode 100644 index 0000000..e2ae680 --- /dev/null +++ b/tools/project-history-migrator-tool/README.md @@ -0,0 +1,89 @@ +# Project History Migrator Tool +This is a CLI utility that allows the migration of projects along with their full version history from a self-managed instance of Intelligent Advisor to a cloud instance. The tool migrates both Policy Modeling projects and Decision Service projects. + +## Prerequisites +### System requirements +- The tool must be run on a machine that has the Java 8 Runtime Environment (JRE) or newer installed +- For running in export mode - network access (JDBC) to the database associated with your self-managed Intelligent Advisor installation is required +- For running in import mode - network access (HTTP) to the Intelligent Advisor cloud instance is required + +### Pre-migration setup +Before beginning the migration process for the project history, the following steps must be taken: +- Migrate users to the cloud instance using the [Users REST API](https://docs.oracle.com/en/cloud/saas/b2c-service/opawx/using-users-rest-api.html), or by creating manually +- Migrate workspaces to the cloud instance using the [Workspaces REST API](https://docs.oracle.com/en/cloud/saas/b2c-service/opawx/using-workspaces-rest-api.html), or by creating manually +- Migrate connections to the cloud instance using the [Connections REST API](https://docs.oracle.com/en/cloud/saas/b2c-service/opawx/using-connections-rest-api.html), or by creating manually +- On the cloud instance, create deployments for any decision service projects that are used as references in other projects +- On the cloud instance, create an API client to use for importing project history. This API client must have both the **manager** and **author** roles for **all workspaces** + +## Building the tool +The tool can be built from source using the Gradle wrapper: + +```bash +# macOS/Linux +./gradlew clean build + +# Windows (PowerShell/CMD) +gradlew.bat clean build +``` + +The JAR will be created under `build/libs/`. It is named after the project directory: +``` +build/libs/project-history-migrator-tool.jar +``` + +## Usage +### Export mode +Usage: +```bash +java -jar project-history-migrator-tool.jar --export [timeZoneId] +``` +Parameters: +- `dbUrl`: JDBC URL that references the MySQL or Oracle database associated with your self-managed Intelligent Advisor installation. Examples `jdbc:mysql://hostname:3306/schema_name`, `jdbc:oracle:thin:@//hostname:1521/PDBNAME` +- `timeZoneId` (optional): Sets the time zone for interpreting and formatting timestamps values read from the database. Examples: 'UTC', 'Europe/London', '+10:00'. Defaults to system time zone. See Java 8 Javadoc ZoneId.of(String): + +Output: +A zip archive containing the exported data, with filename `export-.zip` + +### Import mode (with resume) +Usage: +```bash +java -jar project-history-migrator-tool.jar --import [resumeJournalPath] +``` + +Parameters: +- ``: Base host URL of the Intelligent Advisor site, e.g. `https://name.custhelp.com` +- ``: Path to the exported zip created by this tool +- `[resumeJournalPath]` (optional): Path to a previously written journal file used to resume a previous failed import of project history + +Output: +A journal file in JSON format, with filename `import-.json`. In the event of a failure part way through the import process, the journal file can be used to resume the process. To resume, pass the path to this file as the "resumeJournalPath" parameter. + +## Troubleshooting +- Database connection failed: + - Error: "Failed to connect to the database: ..." + - Verify network, credentials, and JDBC URL. + +- Name clashes on IA Hub: + - Error: "Projects with the following names already exist on the IA Hub: ..." + - Rename or remove conflicting projects on the Hub, or import into a different environment/workspace. + +- Missing workspaces: + - Error: "Target hub is missing workspaces: ..." + - Create the required workspaces on the IA Hub, then retry. + +- OAuth errors: + - Error: "Authentication failed. HTTP code: " + - Or: "Authentication succeeded but access_token not found" + - Verify client id/secret and the Hub URL. + +- HTTP upload failures: + - Error includes the HTTP code and response body. Check that your client has sufficient permissions. + +- Snapshot not found in zip: + - Error: "Snapshot file not found in zip: " + - Ensure the payload zip was produced by this tool and not modified. + +- Resume validation errors: + - "Journal file does not match payload" + - "Journal file does not match specified IA Hub" + - Ensure the correct journal file is provided for the payload and target Hub. diff --git a/tools/project-history-migrator-tool/build.gradle b/tools/project-history-migrator-tool/build.gradle new file mode 100644 index 0000000..6e5800f --- /dev/null +++ b/tools/project-history-migrator-tool/build.gradle @@ -0,0 +1,35 @@ +plugins { + id 'application' +} + +mainClassName = 'com.oracle.determinations.migration.Main' + +java { + sourceCompatibility = JavaVersion.VERSION_1_8 + targetCompatibility = JavaVersion.VERSION_1_8 +} + +repositories { + mavenCentral() +} + +dependencies { + implementation 'com.mysql:mysql-connector-j:8.3.0' // correct coords for MySQL 8.x + implementation 'com.oracle.database.jdbc:ojdbc8:21.12.0.0' // Java 8-compatible Oracle JDBC + implementation 'org.json:json:20240303' + implementation 'org.apache.httpcomponents:httpclient:4.5.14' + implementation 'commons-codec:commons-codec:1.19.0' + + testImplementation 'junit:junit:4.13.2' +} + +// Optional: build a self-contained runnable (fat) JAR +jar { + manifest { + attributes 'Main-Class': mainClassName + } + from { + configurations.runtimeClasspath.collect { it.isDirectory() ? it : zipTree(it) } + } + duplicatesStrategy = DuplicatesStrategy.EXCLUDE +} \ No newline at end of file diff --git a/tools/project-history-migrator-tool/gradle/wrapper/gradle-wrapper.jar b/tools/project-history-migrator-tool/gradle/wrapper/gradle-wrapper.jar new file mode 100644 index 0000000..a4b76b9 Binary files /dev/null and b/tools/project-history-migrator-tool/gradle/wrapper/gradle-wrapper.jar differ diff --git a/tools/project-history-migrator-tool/gradle/wrapper/gradle-wrapper.properties b/tools/project-history-migrator-tool/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 0000000..cea7a79 --- /dev/null +++ b/tools/project-history-migrator-tool/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,7 @@ +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-8.12-bin.zip +networkTimeout=10000 +validateDistributionUrl=true +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists diff --git a/tools/project-history-migrator-tool/gradlew b/tools/project-history-migrator-tool/gradlew new file mode 100755 index 0000000..f3b75f3 --- /dev/null +++ b/tools/project-history-migrator-tool/gradlew @@ -0,0 +1,251 @@ +#!/bin/sh + +# +# Copyright © 2015-2021 the original authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# + +############################################################################## +# +# Gradle start up script for POSIX generated by Gradle. +# +# Important for running: +# +# (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is +# noncompliant, but you have some other compliant shell such as ksh or +# bash, then to run this script, type that shell name before the whole +# command line, like: +# +# ksh Gradle +# +# Busybox and similar reduced shells will NOT work, because this script +# requires all of these POSIX shell features: +# * functions; +# * expansions «$var», «${var}», «${var:-default}», «${var+SET}», +# «${var#prefix}», «${var%suffix}», and «$( cmd )»; +# * compound commands having a testable exit status, especially «case»; +# * various built-in commands including «command», «set», and «ulimit». +# +# Important for patching: +# +# (2) This script targets any POSIX shell, so it avoids extensions provided +# by Bash, Ksh, etc; in particular arrays are avoided. +# +# The "traditional" practice of packing multiple parameters into a +# space-separated string is a well documented source of bugs and security +# problems, so this is (mostly) avoided, by progressively accumulating +# options in "$@", and eventually passing that to Java. +# +# Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS, +# and GRADLE_OPTS) rely on word-splitting, this is performed explicitly; +# see the in-line comments for details. +# +# There are tweaks for specific operating systems such as AIX, CygWin, +# Darwin, MinGW, and NonStop. +# +# (3) This script is generated from the Groovy template +# https://github.com/gradle/gradle/blob/HEAD/platforms/jvm/plugins-application/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt +# within the Gradle project. +# +# You can find Gradle at https://github.com/gradle/gradle/. +# +############################################################################## + +# Attempt to set APP_HOME + +# Resolve links: $0 may be a link +app_path=$0 + +# Need this for daisy-chained symlinks. +while + APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path + [ -h "$app_path" ] +do + ls=$( ls -ld "$app_path" ) + link=${ls#*' -> '} + case $link in #( + /*) app_path=$link ;; #( + *) app_path=$APP_HOME$link ;; + esac +done + +# This is normally unused +# shellcheck disable=SC2034 +APP_BASE_NAME=${0##*/} +# Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036) +APP_HOME=$( cd -P "${APP_HOME:-./}" > /dev/null && printf '%s\n' "$PWD" ) || exit + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD=maximum + +warn () { + echo "$*" +} >&2 + +die () { + echo + echo "$*" + echo + exit 1 +} >&2 + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +nonstop=false +case "$( uname )" in #( + CYGWIN* ) cygwin=true ;; #( + Darwin* ) darwin=true ;; #( + MSYS* | MINGW* ) msys=true ;; #( + NONSTOP* ) nonstop=true ;; +esac + +CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD=$JAVA_HOME/jre/sh/java + else + JAVACMD=$JAVA_HOME/bin/java + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD=java + if ! command -v java >/dev/null 2>&1 + then + die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +fi + +# Increase the maximum file descriptors if we can. +if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then + case $MAX_FD in #( + max*) + # In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC2039,SC3045 + MAX_FD=$( ulimit -H -n ) || + warn "Could not query maximum file descriptor limit" + esac + case $MAX_FD in #( + '' | soft) :;; #( + *) + # In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC2039,SC3045 + ulimit -n "$MAX_FD" || + warn "Could not set maximum file descriptor limit to $MAX_FD" + esac +fi + +# Collect all arguments for the java command, stacking in reverse order: +# * args from the command line +# * the main class name +# * -classpath +# * -D...appname settings +# * --module-path (only if needed) +# * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables. + +# For Cygwin or MSYS, switch paths to Windows format before running java +if "$cygwin" || "$msys" ; then + APP_HOME=$( cygpath --path --mixed "$APP_HOME" ) + CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" ) + + JAVACMD=$( cygpath --unix "$JAVACMD" ) + + # Now convert the arguments - kludge to limit ourselves to /bin/sh + for arg do + if + case $arg in #( + -*) false ;; # don't mess with options #( + /?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath + [ -e "$t" ] ;; #( + *) false ;; + esac + then + arg=$( cygpath --path --ignore --mixed "$arg" ) + fi + # Roll the args list around exactly as many times as the number of + # args, so each arg winds up back in the position where it started, but + # possibly modified. + # + # NB: a `for` loop captures its iteration list before it begins, so + # changing the positional parameters here affects neither the number of + # iterations, nor the values presented in `arg`. + shift # remove old arg + set -- "$@" "$arg" # push replacement arg + done +fi + + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' + +# Collect all arguments for the java command: +# * DEFAULT_JVM_OPTS, JAVA_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments, +# and any embedded shellness will be escaped. +# * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be +# treated as '${Hostname}' itself on the command line. + +set -- \ + "-Dorg.gradle.appname=$APP_BASE_NAME" \ + -classpath "$CLASSPATH" \ + org.gradle.wrapper.GradleWrapperMain \ + "$@" + +# Stop when "xargs" is not available. +if ! command -v xargs >/dev/null 2>&1 +then + die "xargs is not available" +fi + +# Use "xargs" to parse quoted args. +# +# With -n1 it outputs one arg per line, with the quotes and backslashes removed. +# +# In Bash we could simply go: +# +# readarray ARGS < <( xargs -n1 <<<"$var" ) && +# set -- "${ARGS[@]}" "$@" +# +# but POSIX shell has neither arrays nor command substitution, so instead we +# post-process each arg (as a line of input to sed) to backslash-escape any +# character that might be a shell metacharacter, then use eval to reverse +# that process (while maintaining the separation between arguments), and wrap +# the whole thing up as a single "set" statement. +# +# This will of course break if any of these variables contains a newline or +# an unmatched quote. +# + +eval "set -- $( + printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" | + xargs -n1 | + sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' | + tr '\n' ' ' + )" '"$@"' + +exec "$JAVACMD" "$@" diff --git a/tools/project-history-migrator-tool/gradlew.bat b/tools/project-history-migrator-tool/gradlew.bat new file mode 100644 index 0000000..9b42019 --- /dev/null +++ b/tools/project-history-migrator-tool/gradlew.bat @@ -0,0 +1,94 @@ +@rem +@rem Copyright 2015 the original author or authors. +@rem +@rem Licensed under the Apache License, Version 2.0 (the "License"); +@rem you may not use this file except in compliance with the License. +@rem You may obtain a copy of the License at +@rem +@rem https://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem +@rem SPDX-License-Identifier: Apache-2.0 +@rem + +@if "%DEBUG%"=="" @echo off +@rem ########################################################################## +@rem +@rem Gradle startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables with windows NT shell +if "%OS%"=="Windows_NT" setlocal + +set DIRNAME=%~dp0 +if "%DIRNAME%"=="" set DIRNAME=. +@rem This is normally unused +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Resolve any "." and ".." in APP_HOME to make it shorter. +for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if %ERRORLEVEL% equ 0 goto execute + +echo. 1>&2 +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 1>&2 +echo. 1>&2 +echo Please set the JAVA_HOME variable in your environment to match the 1>&2 +echo location of your Java installation. 1>&2 + +goto fail + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto execute + +echo. 1>&2 +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 1>&2 +echo. 1>&2 +echo Please set the JAVA_HOME variable in your environment to match the 1>&2 +echo location of your Java installation. 1>&2 + +goto fail + +:execute +@rem Setup the command line + +set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar + + +@rem Execute Gradle +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* + +:end +@rem End local scope for the variables with windows NT shell +if %ERRORLEVEL% equ 0 goto mainEnd + +:fail +rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of +rem the _cmd.exe /c_ return code! +set EXIT_CODE=%ERRORLEVEL% +if %EXIT_CODE% equ 0 set EXIT_CODE=1 +if not ""=="%GRADLE_EXIT_CONSOLE%" exit %EXIT_CODE% +exit /b %EXIT_CODE% + +:mainEnd +if "%OS%"=="Windows_NT" endlocal + +:omega diff --git a/tools/project-history-migrator-tool/src/main/java/com/oracle/determinations/migration/DateTimeUtil.java b/tools/project-history-migrator-tool/src/main/java/com/oracle/determinations/migration/DateTimeUtil.java new file mode 100644 index 0000000..233bc33 --- /dev/null +++ b/tools/project-history-migrator-tool/src/main/java/com/oracle/determinations/migration/DateTimeUtil.java @@ -0,0 +1,48 @@ +package com.oracle.determinations.migration; + +import java.time.Instant; +import java.time.OffsetDateTime; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeParseException; + +/** + * Utilities for comparing ISO_OFFSET_DATE_TIME strings (e.g., 2025-11-04T12:34:56+10:00 or 2025-11-04T12:34:56Z). + * Compares equality by Instant, accounting for timezone. + */ +public final class DateTimeUtil { + + private static final DateTimeFormatter ISO_OFFSET = DateTimeFormatter.ISO_OFFSET_DATE_TIME; + + private DateTimeUtil() {} + + /** + * Returns true if both date strings represent the same instant in time (accounting for timezone). + * - If both are null: true + * - If one is null: false + * - Tries to parse both with ISO_OFFSET_DATE_TIME; if either is unparsable, treats them as equal (lenient) + * - Otherwise compares their Instants + */ + public static boolean sameInstant(String a, String b) { + if (a == null && b == null) return true; + if (a == null || b == null) return false; + + Instant ia = instantOrNull(a); + Instant ib = instantOrNull(b); + if (ia == null || ib == null) { + return true; + } + return ia.equals(ib); + } + + /** + * Returns the Instant represented by the given date string, or null if it cannot be parsed. + */ + public static Instant instantOrNull(String s) { + if (s == null) return null; + try { + return OffsetDateTime.parse(s, ISO_OFFSET).toInstant(); + } catch (DateTimeParseException ex) { + return null; + } + } +} diff --git a/tools/project-history-migrator-tool/src/main/java/com/oracle/determinations/migration/DecisionServiceProject.java b/tools/project-history-migrator-tool/src/main/java/com/oracle/determinations/migration/DecisionServiceProject.java new file mode 100644 index 0000000..ed70359 --- /dev/null +++ b/tools/project-history-migrator-tool/src/main/java/com/oracle/determinations/migration/DecisionServiceProject.java @@ -0,0 +1,41 @@ +package com.oracle.determinations.migration; + +import org.json.JSONObject; + +/** + * Represents a decision service project with optional source linkage and workspace. + */ +public class DecisionServiceProject { + public final String projectName; + public final String workspace; + public final String fromProjectName; + public final Integer fromVersionNumber; + + /** + * Constructs a decision service project descriptor. + * @param projectName Decision service project name. + * @param workspace Workspace the decision service project belongs to. + * @param fromProjectName Optional source project name for initial version cloning. + * @param fromVersionNumber Optional source project version number for cloning. + */ + public DecisionServiceProject(String projectName, String workspace, String fromProjectName, Integer fromVersionNumber) { + this.projectName = projectName; + this.workspace = workspace; + this.fromProjectName = fromProjectName; + this.fromVersionNumber = fromVersionNumber; + } + + /** + * Builds a decision service project from a JSON object. + * @param obj JSON with keys: module_name, workspace, and optional from_module_name, from_version_number. + * @return parsed DecisionServiceProject instance. + * @throws org.json.JSONException if required fields are missing or invalid. + */ + public static DecisionServiceProject fromJson(JSONObject obj) { + String projectName = obj.getString("module_name"); + String workspace = obj.getString("workspace"); + String fromProjectName = obj.has("from_module_name") ? obj.getString("from_module_name") : null; + Integer fromVersionNumber = obj.has("from_version_number") ? obj.getInt("from_version_number") : null; + return new DecisionServiceProject(projectName, workspace, fromProjectName, fromVersionNumber); + } +} diff --git a/tools/project-history-migrator-tool/src/main/java/com/oracle/determinations/migration/DecisionServiceProjectVersion.java b/tools/project-history-migrator-tool/src/main/java/com/oracle/determinations/migration/DecisionServiceProjectVersion.java new file mode 100644 index 0000000..c4d1e1c --- /dev/null +++ b/tools/project-history-migrator-tool/src/main/java/com/oracle/determinations/migration/DecisionServiceProjectVersion.java @@ -0,0 +1,148 @@ +package com.oracle.determinations.migration; + +import org.json.JSONObject; +import java.util.Objects; + +/** + * Represents a decision service version and related metadata. + */ +public class DecisionServiceProjectVersion implements ProjectVersion { + public final String projectName; + public final int versionNumber; + public final String createTimestamp; + public final int imported; + public final String userName; + public final String definition; + public final String description; + public final String descriptionUpdated; + public final String descriptionAuthor; + public final String fingerprintSha256; + public final boolean isDraft; + + /** + * Creates a new decision service project version. + * @param projectName decision service project name. + * @param versionNumber version number (0 indicates draft). + * @param createTimestamp creation timestamp. + * @param imported import flag/counter from source DB. + * @param userName author of the version. + * @param definition decision service project definition content (JSON string). + * @param description optional description. + * @param descriptionUpdated optional description last-updated timestamp. + * @param descriptionAuthor optional description author. + * @param fingerprintSha256 definition fingerprint hash. + */ + public DecisionServiceProjectVersion(String projectName, + int versionNumber, + String createTimestamp, + int imported, + String userName, + String definition, + String description, + String descriptionUpdated, + String descriptionAuthor, + String fingerprintSha256) { + this.projectName = projectName; + this.versionNumber = versionNumber; + this.createTimestamp = createTimestamp; + this.imported = imported; + this.userName = userName; + this.definition = definition; + this.description = description; + this.descriptionUpdated = descriptionUpdated; + this.descriptionAuthor = descriptionAuthor; + this.fingerprintSha256 = fingerprintSha256; + this.isDraft = versionNumber == 0; + } + + /** + * Builds an instance from a JSON object. + * @param obj JSON with keys: module_name, version_number, create_timestamp, module_imported, user_name, definition; optional description, description_updated, description_author, fingerprint_sha256. + * @return parsed DecisionServiceVersion. + * @throws org.json.JSONException if required fields are missing or invalid. + */ + public static DecisionServiceProjectVersion fromJson(JSONObject obj) { + String projectName = obj.getString("module_name"); + int versionNumber = obj.getInt("version_number"); + String createTimestamp = obj.getString("create_timestamp"); + int imported = obj.getInt("module_imported"); + String userName = obj.getString("user_name"); + String definition = obj.getString("definition"); + String description = obj.has("description") ? obj.getString("description") : null; + String descriptionUpdated = obj.has("description_updated") ? obj.getString("description_updated") : null; + String descriptionAuthor = obj.has("description_author") ? obj.getString("description_author") : null; + String fingerprint = obj.getString("fingerprint_sha256"); + return new DecisionServiceProjectVersion(projectName, versionNumber, createTimestamp, imported, userName, definition, + description, descriptionUpdated, descriptionAuthor, fingerprint); + } + + + /** + * Serializes a minimal JSON entry for the import/export journal. + * @param index sequential index for the journal entry. + * @return JSON object for the journal. + */ + public JSONObject toJSONForJournal(int index) { + JSONObject obj = new JSONObject(); + obj.put("index", index); + obj.put("project_name", projectName); + obj.put("version_number", versionNumber); + obj.put("type", "decision"); + obj.put("sha256", fingerprintSha256); + return obj; + } + + /** + * Value equality based on key fields. + */ + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof DecisionServiceProjectVersion)) return false; + DecisionServiceProjectVersion that = (DecisionServiceProjectVersion) o; + return versionNumber == that.versionNumber + && Objects.equals(projectName, that.projectName) + && DateTimeUtil.sameInstant(createTimestamp, that.createTimestamp) + && Objects.equals(userName, that.userName) + && Objects.equals(description, that.description) + && DateTimeUtil.sameInstant(descriptionUpdated, that.descriptionUpdated) + && Objects.equals(descriptionAuthor, that.descriptionAuthor) + && Objects.equals(fingerprintSha256, that.fingerprintSha256); + } + + /** + * Hash code consistent with equals. + */ + @Override + public int hashCode() { + return Objects.hash(projectName, versionNumber, + DateTimeUtil.instantOrNull(createTimestamp), + userName, definition, description, + DateTimeUtil.instantOrNull(descriptionUpdated), + descriptionAuthor, fingerprintSha256); + } + + /** + * Returns the version number. + */ + @Override + public int getVersion() { + return versionNumber; + } + + /** + * Returns the project name. + */ + @Override + public String getProjectName() { + return projectName; + } + + /** + * Indicates whether this version is a draft. + */ + @Override + public boolean isDraft() { + return isDraft; + } +} diff --git a/tools/project-history-migrator-tool/src/main/java/com/oracle/determinations/migration/Exporter.java b/tools/project-history-migrator-tool/src/main/java/com/oracle/determinations/migration/Exporter.java new file mode 100644 index 0000000..7543ab2 --- /dev/null +++ b/tools/project-history-migrator-tool/src/main/java/com/oracle/determinations/migration/Exporter.java @@ -0,0 +1,407 @@ +package com.oracle.determinations.migration; + +import java.sql.*; + +import org.json.JSONArray; +import org.json.JSONObject; +import java.util.Base64; + +import java.util.Calendar; +import java.util.TimeZone; + +import java.io.IOException; +import java.time.Instant; +import java.time.ZoneId; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; +import java.util.HashSet; +import java.util.Set; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; + +/** + * Exports projects, decision service projects, versions, and snapshots from the database into a zip. + */ +public class Exporter { + + private Set exportedSnapshotIds; + + private final ZoneId zoneId; + private final Calendar tzCalendar; + + public Exporter() { + this(ZoneId.systemDefault()); + } + + public Exporter(ZoneId zoneId) { + this.zoneId = (zoneId != null) ? zoneId : ZoneId.systemDefault(); + this.tzCalendar = Calendar.getInstance(TimeZone.getTimeZone(this.zoneId)); + } + + /** + * Establishes a JDBC connection. + * @param dbUrl JDBC URL (supports jdbc:mysql: and jdbc:oracle:). + * @param username DB username. + * @param password DB password. + * @return open JDBC connection. + * @throws SQLException on connection errors. + * @throws ClassNotFoundException if the JDBC driver class is not found. + */ + public Connection establishConnection(String dbUrl, String username, String password) throws SQLException, ClassNotFoundException { + String urlLower = dbUrl == null ? "" : dbUrl.toLowerCase(); + if (urlLower.startsWith("jdbc:mysql:")) { + Class.forName("com.mysql.cj.jdbc.Driver"); + } else if (urlLower.startsWith("jdbc:oracle:")) { + Class.forName("oracle.jdbc.OracleDriver"); + } else { + throw new IllegalArgumentException("Unrecognized JDBC URL scheme: " + dbUrl + ". Supported schemes are jdbc:mysql: and jdbc:oracle:"); + } + return DriverManager.getConnection(dbUrl, username, password); + } + + /** + * Performs the export and writes JSON files and snapshots to the zip. + * @param connection JDBC connection to read from. + * @param zos target zip stream to write entries into. + * @throws RuntimeException if any step fails (wraps underlying exceptions). + */ + public void doExport(Connection connection, ZipOutputStream zos) { + exportedSnapshotIds = new HashSet<>(); + try { + String exportedProjects = exportProjects(connection); + addZipEntry(zos, exportedProjects, "projects.json"); + String exportedProjectVersions = exportProjectVersions(connection, zos); + addZipEntry(zos, exportedProjectVersions, "project_versions.json"); + + // Export decision service projects and versions + String exportedDecisionServiceProjects = exportDecisionServiceProjects(connection); + addZipEntry(zos, exportedDecisionServiceProjects, "modules.json"); + String exportedDecisionServiceProjectVersions = exportDecisionServiceProjectVersions(connection); + addZipEntry(zos, exportedDecisionServiceProjectVersions, "module_versions.json"); + + } catch (Exception ex) { + throw new RuntimeException(ex.getMessage(), ex); + } + } + + /** + * Generates a timestamped export zip filename. + * @return file name for the export zip. + */ + protected String newExportZipFileName() { + return "export-" + System.currentTimeMillis() + ".zip"; + } + + /** + * Formats a SQL timestamp to ISO-8601 with zone offset. + * @param timestamp SQL timestamp (nullable). + * @return formatted timestamp or null. + */ + private String formatTimestamp(Timestamp timestamp) { + if (timestamp == null) { + return null; + } + Instant instant = timestamp.toInstant(); + ZoneId zoneId = this.zoneId; + ZonedDateTime zdt = ZonedDateTime.ofInstant(instant, zoneId); + return zdt.format(DateTimeFormatter.ISO_OFFSET_DATE_TIME); + } + + // Returns project versions JSON, each with three child arrays for inclusions, changes, and decision refs + /** + * Exports project versions with inclusions, changes, and decision refs. + * @param connection JDBC connection. + * @param zos zip output stream (used to add snapshot binaries). + * @return JSON array string of project versions. + * @throws Exception on SQL or zip I/O errors. + */ + private String exportProjectVersions(Connection connection, ZipOutputStream zos) throws Exception { + JSONArray versions = new JSONArray(); + String versionSql = "SELECT pv.project_version_id, p.project_id, p.project_name, pv.project_version, pv.project_snapshot_id, pv.version_uuid, pv.activatable, pv.user_name,\n" + + "pv.opa_version, pv.description, pv.creation_date, pv.deleted_timestamp, pv.project_inclusions, pv.description_updated,\n" + + "pv.description_author, pv.runtime_dependency_ref_type FROM PROJECT_VERSION pv, PROJECT p WHERE p.project_id = pv.project_id AND pv.deleted_timestamp IS NULL ORDER BY pv.project_version_id ASC"; + try ( + Statement versionStmt = connection.createStatement(); + ResultSet versionRs = versionStmt.executeQuery(versionSql) + ) { + while (versionRs.next()) { + JSONObject versionJson = new JSONObject(); + int projectVersionId = versionRs.getInt(1); + int projectSnapshotId = versionRs.getInt(5); + + String projectName = versionRs.getString(3); + int projectVersionNumber = versionRs.getInt(4); + System.out.println("Exporting project version: " + projectName + " (version " + projectVersionNumber + ")"); + + // Serialize all columns of PROJECT_VERSION + versionJson.put("project_name", projectName); + versionJson.put("project_version_number", projectVersionNumber); + versionJson.put("user_name", versionRs.getString(8)); + versionJson.put("opa_version", versionRs.getString(9)); + versionJson.put("description", versionRs.getString(10)); + versionJson.put("creation_date", formatTimestamp(versionRs.getTimestamp(11, tzCalendar))); + versionJson.put("description_updated", formatTimestamp(versionRs.getTimestamp(14, tzCalendar))); + versionJson.put("description_author", versionRs.getString(15)); + + // CHILD ARRAY 1: PROJECT_VERSION_INCLUSION + JSONArray inclusions = new JSONArray(); + String inclSql = "SELECT (SELECT p.project_name FROM PROJECT p, PROJECT_VERSION pv WHERE PROJECT_VERSION_INCLUSION.included_version_id = pv.project_version_id AND p.project_id = pv.project_id) AS included_project_name,\n" + + "(SELECT project_version FROM PROJECT_VERSION pv WHERE PROJECT_VERSION_INCLUSION.included_version_id = pv.project_version_id) AS included_project_version_number \n" + + "FROM PROJECT_VERSION_INCLUSION WHERE project_version_id = ?"; + try (java.sql.PreparedStatement ps = connection.prepareStatement(inclSql)) { + ps.setInt(1, projectVersionId); + try (ResultSet rs = ps.executeQuery()) { + while (rs.next()) { + JSONObject incl = new JSONObject(); + incl.put("included_project_name", rs.getString("included_project_name")); + incl.put("included_project_version_number", rs.getString("included_project_version_number")); + inclusions.put(incl); + } + } + } + versionJson.put("project_version_inclusions", inclusions); + + // CHILD ARRAY 2: PROJECT_VERSION_CHANGE + JSONArray changes = new JSONArray(); + String changeSql = "SELECT object_name, change_type FROM PROJECT_VERSION_CHANGE WHERE project_version_id = ?"; + try (java.sql.PreparedStatement ps = connection.prepareStatement(changeSql)) { + ps.setInt(1, projectVersionId); + try (ResultSet rs = ps.executeQuery()) { + while (rs.next()) { + JSONObject change = new JSONObject(); + change.put("object_name", rs.getString("object_name")); + + int changeType = rs.getInt("change_type"); + + switch (changeType) { + case 0: + change.put("change_type", "add"); + break; + case 1: + change.put("change_type", "delete"); + break; + case 2: + change.put("change_type", "modify"); + break; + default: + throw new IllegalStateException("Unexpected change type: " + changeType); + } + + changes.put(change); + } + } + } + versionJson.put("project_version_changes", changes); + + // CHILD ARRAY 3: PROJECT_DECISION_REFS + JSONArray decisionRefs = new JSONArray(); + String refsSql = "SELECT * FROM PROJECT_DECISION_REFS WHERE project_version_id = ?"; + try (java.sql.PreparedStatement ps = connection.prepareStatement(refsSql)) { + ps.setInt(1, projectVersionId); + try (ResultSet rs = ps.executeQuery()) { + while (rs.next()) { + JSONObject ref = new JSONObject(); + ref.put("decision_service_ref", rs.getString("decision_service_ref")); + ref.put("project_version_id", rs.getInt("project_version_id")); + decisionRefs.put(ref); + } + } + } + versionJson.put("project_decision_refs", decisionRefs); + + //SNAPSHOT + String fingerprintSha256 = null; + String snapshotSql = "SELECT fingerprint_sha256, uploaded_date FROM SNAPSHOT WHERE snapshot_id = ?"; + try (java.sql.PreparedStatement ps = connection.prepareStatement(snapshotSql)) { + ps.setInt(1, projectSnapshotId); + + try (ResultSet rs = ps.executeQuery()) { + if (rs.next()) { + fingerprintSha256 = rs.getString("fingerprint_sha256"); + } + } + } + versionJson.put("fingerprint_sha256", fingerprintSha256); + + if (fingerprintSha256 != null && !exportedSnapshotIds.contains(fingerprintSha256)) { + exportProjectSnapshot(connection, projectSnapshotId, fingerprintSha256, zos); + exportedSnapshotIds.add(fingerprintSha256); + } + + versions.put(versionJson); + } + } + return versions.toString(); + } + + + /** + * Exports all projects as projects.json content. + * @param connection JDBC connection. + * @return JSON array string of projects. + * @throws SQLException on query errors. + */ + private String exportProjects(Connection connection) throws SQLException { + JSONArray jsonArray = new JSONArray(); + String sql = "SELECT project_name," + + " (SELECT p.project_name FROM PROJECT p, PROJECT_VERSION pv WHERE PROJECT.from_project_version_id = pv.project_version_id AND p.project_id = pv.project_id) AS from_project_name," + + " (SELECT pv.project_version FROM PROJECT_VERSION pv WHERE PROJECT.from_project_version_id = pv.project_version_id) AS from_project_version_number," + + " (SELECT collection_name FROM COLLECTION WHERE collection_id IN (SELECT collection_id FROM PROJECT_COLL WHERE project_id = PROJECT.project_id)) as workspace" + + " FROM PROJECT WHERE deleted_timestamp IS NULL ORDER BY project_id ASC"; + Statement stmt = connection.createStatement(); + ResultSet rs = stmt.executeQuery(sql); + while (rs.next()) { + JSONObject jo = new JSONObject(); + String projectName = rs.getString("project_name"); + System.out.println("Exporting project: " + projectName); + jo.put("project_name", projectName); + jo.put("from_project_name", rs.getObject("from_project_name")); + jo.put("from_project_version_number", rs.getObject("from_project_version_number")); + jo.put("workspace", rs.getObject("workspace")); + jsonArray.put(jo); + } + return jsonArray.toString(); + } + + // Export all decision service projects as modules.json + /** + * Exports decision service projects as modules.json content. + * @param connection JDBC connection. + * @return JSON array string of decision service projects. + * @throws SQLException on query errors. + */ + private String exportDecisionServiceProjects(Connection connection) throws SQLException { + JSONArray jsonArray = new JSONArray(); + String sql = "SELECT m.module_id, m.module_name, m.module_kind," + + "(SELECT module_name FROM MODULE WHERE module_id = mv.module_id) AS from_module_name," + + "mv.version_number AS from_version_number " + + "FROM MODULE m " + + "LEFT JOIN MODULE_VERSION mv ON m.module_from_version_id = mv.module_version_id ORDER BY m.module_id ASC"; + try (Statement stmt = connection.createStatement(); + ResultSet rs = stmt.executeQuery(sql)) { + while (rs.next()) { + JSONObject jo = new JSONObject(); + int moduleId = rs.getInt("module_id"); + String moduleName = rs.getString("module_name"); + System.out.println("Exporting decision service project: " + moduleName); + jo.put("module_name", moduleName); + jo.put("module_kind", rs.getInt("module_kind")); + + // Optionally, add source module/version info if present + String fromModuleName = rs.getString("from_module_name"); + if (!rs.wasNull()) { + jo.put("from_module_name", fromModuleName); + } + int fromVersionNumber = rs.getInt("from_version_number"); + if (!rs.wasNull()) { + jo.put("from_version_number", fromVersionNumber); + } + + // Add collections ("workspaces") for this module + String collSql = "SELECT c.collection_name FROM MODULE_COLL mc, COLLECTION c WHERE mc.collection_id = c.collection_id AND mc.module_id = ?"; + try (PreparedStatement ps = connection.prepareStatement(collSql)) { + ps.setInt(1, moduleId); + try (ResultSet collRs = ps.executeQuery()) { + while (collRs.next()) { + jo.put("workspace", collRs.getString("collection_name")); + } + } + } + + jsonArray.put(jo); + } + } + return jsonArray.toString(); + } + + // Export all decision service project versions as module_versions.json + /** + * Exports decision service project versions as module_versions.json content. + * @param connection JDBC connection. + * @return JSON array string of decision service project versions. + * @throws SQLException on query errors. + */ + private String exportDecisionServiceProjectVersions(Connection connection) throws SQLException { + JSONArray jsonArray = new JSONArray(); + String sql = "SELECT mv.module_version_id, mv.module_id, m.module_name, mv.version_number, mv.format_version, mv.create_timestamp, mv.user_name, " + + "mv.fingerprint_sha256, mv.definition, mv.module_imported, mv.description, mv.description_updated, mv.description_author " + + "FROM MODULE_VERSION mv JOIN MODULE m ON mv.module_id = m.module_id ORDER BY mv.module_version_id ASC"; + try (Statement stmt = connection.createStatement(); + ResultSet rs = stmt.executeQuery(sql)) { + while (rs.next()) { + JSONObject jo = new JSONObject(); + + String moduleName = rs.getString("module_name"); + int versionNumber = rs.getInt("version_number"); + System.out.println("Exporting decision service project version: " + moduleName + (versionNumber == 0 ? " (draft)" : " (version " + versionNumber + ")")); + + jo.put("module_name", moduleName); + jo.put("version_number", versionNumber); + jo.put("create_timestamp", formatTimestamp(rs.getTimestamp("create_timestamp", tzCalendar))); + jo.put("user_name", rs.getString("user_name")); + jo.put("fingerprint_sha256", rs.getString("fingerprint_sha256")); + jo.put("definition", rs.getString("definition")); + jo.put("module_imported", rs.getInt("module_imported")); + jo.put("description", rs.getString("description")); + jo.put("description_updated", formatTimestamp(rs.getTimestamp("description_updated", tzCalendar))); + jo.put("description_author", rs.getString("description_author")); + + jsonArray.put(jo); + } + } + return jsonArray.toString(); + } + + /** + * Adds a UTF-8 text file entry with content to the zip archive. + * @param zos target zip stream. + * @param content file contents (UTF-8). + * @param fileName entry name. + * @throws RuntimeException if the entry cannot be written. + */ + private void addZipEntry(ZipOutputStream zos, String content, String fileName) { + try { + ZipEntry entry = new ZipEntry(fileName); + zos.putNextEntry(entry); + byte[] data = content.getBytes("UTF-8"); + zos.write(data, 0, data.length); + zos.closeEntry(); + } catch (IOException e) { + throw new RuntimeException("Failed to add zip entry '" + fileName + "': " + e.getMessage(), e); + } + } + + // Exports a project snapshot by fetching and decoding chunk slices, then writing to zip with given fingerprint as filename + /** + * Writes the decoded snapshot content to the zip using the fingerprint as the filename. + * @param connection JDBC connection. + * @param snapshotId snapshot identifier. + * @param sha256Fingerprint filename to use in the zip. + * @param zos target zip stream. + * @throws Exception on SQL or zip I/O errors. + */ + private void exportProjectSnapshot(Connection connection, int snapshotId, String sha256Fingerprint, ZipOutputStream zos) throws Exception { + String sql = "SELECT chunk_slice FROM SNAPSHOT_CHUNK WHERE snapshot_id = ? ORDER BY chunk_sequence ASC"; + StringBuilder b64Concat = new StringBuilder(); + try (java.sql.PreparedStatement ps = connection.prepareStatement(sql)) { + ps.setInt(1, snapshotId); + try (ResultSet rs = ps.executeQuery()) { + while (rs.next()) { + b64Concat.append(rs.getString("chunk_slice")); + } + } + } + if (b64Concat.length() == 0) { + throw new IOException("No snapshot chunks found for snapshot_id " + snapshotId); + } + // Decode base64 + + byte[] content = Base64.getDecoder().decode(b64Concat.toString()); + ZipEntry entry = new ZipEntry(sha256Fingerprint); + zos.putNextEntry(entry); + zos.write(content, 0, content.length); + zos.closeEntry(); + } + + +} diff --git a/tools/project-history-migrator-tool/src/main/java/com/oracle/determinations/migration/Importer.java b/tools/project-history-migrator-tool/src/main/java/com/oracle/determinations/migration/Importer.java new file mode 100644 index 0000000..a47e121 --- /dev/null +++ b/tools/project-history-migrator-tool/src/main/java/com/oracle/determinations/migration/Importer.java @@ -0,0 +1,802 @@ +package com.oracle.determinations.migration; + +import java.util.Base64; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.entity.StringEntity; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.util.EntityUtils; +import org.apache.http.HttpHeaders; +import org.apache.http.entity.ContentType; +import org.apache.commons.codec.digest.DigestUtils; +import org.apache.http.HttpEntity; + +import java.io.BufferedReader; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.net.URLEncoder; +import java.util.*; + +import java.util.zip.ZipEntry; +import java.util.zip.ZipFile; + +import org.json.JSONArray; +import org.json.JSONObject; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Paths; + +/** + * Imports exported OPA projects and decision service projects into an IA Hub with resume support. + */ +public class Importer { + + private static final String opmProjectUrlPath = "/opa-hub/api/experimental/migrate-opm-project-version"; + private static final String decisionServiceProjectUrlPath = "/opa-hub/api/experimental/migrate-decision-service-project-version"; + private static final String projectVersionsUrlPath = "/opa-hub/api/12.2.39/projects?expand=versions"; + private static final String workspacesUrlPath = "/opa-hub/api/12.2.39/workspaces?links=none&fields=name"; + private static final String authUrlPath = "/opa-hub/api/12.2.39/auth"; + + private final Map opmProjectsByName; + private final Map decisionServiceProjectsByName; + + private String resumePayloadSha256; + private String resumeHubUrl; + + // Testability seams + /** + * Abstraction over HTTP for easier testing. + */ + public interface HttpTransport { + /** + * Executes an HTTP request. + * @param method HTTP method (GET or POST). + * @param url Target URL. + * @param headers Request headers to send. + * @param body Optional request body for POST (UTF-8). + * @return HTTP result with status code and body text. + * @throws Exception if the transport fails or the request cannot be executed. + */ + HttpResult request(String method, String url, Map headers, String body) throws Exception; + } + + @FunctionalInterface + /** + * Factory for creating Journal instances. + */ + public interface JournalFactory { + /** + * Creates a Journal instance. + * @param path File path to write journal entries to. + * @return a Journal for the specified path. + * @throws IOException if the file cannot be created or opened. + */ + Journal create(String path) throws IOException; + } + + private final HttpTransport httpTransport; + private final JournalFactory journalFactory; + + /** + * Creates an Importer with default HTTP transport and journal factory. + */ + public Importer() { + this(new DefaultHttpTransport(), Journal::new); + } + + /** + * Creates an Importer with injected HTTP transport and journal factory. + */ + public Importer(HttpTransport httpTransport, JournalFactory journalFactory) { + this.httpTransport = httpTransport; + this.journalFactory = journalFactory; + this.opmProjectsByName = new HashMap<>(); + this.decisionServiceProjectsByName = new HashMap<>(); + } + + /** + * Imports the provided payload into the target IA Hub, optionally resuming from a journal. + * @param iaHostUrl Base URL of the IA Hub. + * @param iaUsername Client ID/username for OAuth. + * @param iaPassword Client secret/password for OAuth. + * @param exportedPayloadPath Path to the exported zip payload. + * @param resumeJournalPath Optional path to a journal file to resume from. + * @throws Exception on I/O, authentication, HTTP, or validation errors. + */ + public void doImport(String iaHostUrl, String iaUsername, String iaPassword, String exportedPayloadPath, String resumeJournalPath) throws Exception { + + String payloadSha256 = DigestUtils.sha256Hex(Files.newInputStream(Paths.get(exportedPayloadPath))); + + ZipFile zipPayload = new ZipFile(new File(exportedPayloadPath)); + List payloadProjectVersions = parsePayload(zipPayload); + + String oAuthToken = authenticate(iaHostUrl, iaUsername, iaPassword); + + Map> existingProjectVersionsByName = fetchHubVersions(iaHostUrl, oAuthToken); + + // Optional resume from journal + List resumeEntries = null; + int journalItemsLastIndex = -1; + if (resumeJournalPath != null && !resumeJournalPath.isEmpty()) { + resumeEntries = parseJournalEntries(resumeJournalPath); + + if (!resumePayloadSha256.equals(payloadSha256)) { + throw new RuntimeException("Journal file does not match payload"); + } + + if (!resumeHubUrl.equals(iaHostUrl)) { + throw new RuntimeException("Journal file does not match specified IA Hub"); + } + + journalItemsLastIndex = resumeEntries.get(resumeEntries.size() - 1).getInt("index"); + List alreadyUploadedItems = payloadProjectVersions.subList(0, journalItemsLastIndex + 1); + Map> alreadyUploadedVersionsByProjectName = new HashMap<>(); + for (ProjectVersion projectVersion : alreadyUploadedItems) { + alreadyUploadedVersionsByProjectName.computeIfAbsent(projectVersion.getProjectName(), k -> new ArrayList<>()).add(projectVersion); + } + + boolean error = false; + for (String projectName : alreadyUploadedVersionsByProjectName.keySet()) { + if (!existingProjectVersionsByName.containsKey(projectName)) { + error = true; + System.out.println("Error - project " + projectName + " not found not on Hub"); + } else { + List expectedVersions = alreadyUploadedVersionsByProjectName.get(projectName); + List actualVersions = existingProjectVersionsByName.get(projectName); + + if (!expectedVersions.equals(actualVersions)) { + error = true; + System.out.println("Error - IA Hub has version for project " + projectName + " that do not match the payload being imported"); + } + } + } + if (error) { + throw new RuntimeException("Could not resume import - IA Hub has projects/versions that do not match the payload being imported"); + } + } else { + // Fail if there are any project name clashes + Set existingProjectNames = existingProjectVersionsByName.keySet(); + Set clashingProjectNames = new HashSet<>(opmProjectsByName.keySet()); + clashingProjectNames.addAll(decisionServiceProjectsByName.keySet()); + clashingProjectNames.retainAll(existingProjectNames); + + if (!clashingProjectNames.isEmpty()) { + throw new RuntimeException("Projects with the following names already exist on the IA Hub: " + String.join(",", clashingProjectNames)); + } + + // Fail if there are any missing workspaces + Set payloadWorkspacesNames = new HashSet<>(); + for (DecisionServiceProject m : decisionServiceProjectsByName.values()) { + payloadWorkspacesNames.add(m.workspace); + } + for (OPMProject project : opmProjectsByName.values()) { + payloadWorkspacesNames.add(project.workspace); + } + + Set hubWorkspacesNames = fetchWorkspaceNames(iaHostUrl, oAuthToken); + Set missingWorkspaces = new HashSet<>(payloadWorkspacesNames); + missingWorkspaces.removeAll(hubWorkspacesNames); + if (!missingWorkspaces.isEmpty()) { + throw new RuntimeException("Target hub is missing workspaces: " + String.join(",", missingWorkspaces)); + } + } + + // --- Proceed to import --- + String journalFileName = newJournalFileName(); + + try (Journal journal = journalFactory.create(journalFileName)) { + JSONObject journalHeader = new JSONObject(); + journalHeader.put("payload_sha256", payloadSha256); + journalHeader.put("hub_url", iaHostUrl); + journal.write(journalHeader); + + List projectVersionsToImport = new ArrayList<>(payloadProjectVersions); + + int journalIndex = 0; + // Add any entries from the journal file we are using to resume so that we can resume again if the upload process is interrupted again + if (resumeEntries != null) { + for (JSONObject resumeEntry : resumeEntries) { + journal.write(resumeEntry); + journalIndex++; + } + + projectVersionsToImport = projectVersionsToImport.subList(journalIndex, projectVersionsToImport.size()); + + System.out.println("Resuming from journal"); + } + + + for (ProjectVersion projectVersion: projectVersionsToImport) { + if (projectVersion instanceof OPMProjectVersion) { + OPMProjectVersion opmProjectVersion = (OPMProjectVersion) projectVersion; + importOPMProjectVersion(opmProjectVersion, zipPayload, iaHostUrl, oAuthToken); + System.out.println("Imported policy modeling project: " + opmProjectVersion.projectName + " (version " + opmProjectVersion.projectVersionNumber + ")"); + journal.write(opmProjectVersion.toJSONForJournal(journalIndex)); + } else { + DecisionServiceProjectVersion decisionServiceVersion = (DecisionServiceProjectVersion) projectVersion; + importModuleVersion(decisionServiceVersion, iaHostUrl, oAuthToken, journal); + System.out.println("Imported decision service project: " + decisionServiceVersion.projectName + " (version " + (decisionServiceVersion.isDraft ? "draft" : decisionServiceVersion.versionNumber) + ")"); + journal.write(decisionServiceVersion.toJSONForJournal(journalIndex)); + } + + journalIndex++; + } + } + + zipPayload.close(); + } + + /** + * Generates a timestamped journal file name. + */ + protected String newJournalFileName() { + return "import-" + System.currentTimeMillis() + ".json"; + } + + /** + * Parses the payload zip into project/module version objects. + * @param zip The payload zip file. + * @return list of versions to import, in encountered order. + * @throws IOException if zip entries cannot be read. + */ + private List parsePayload(ZipFile zip) throws IOException { + List projectVersions = new ArrayList<>(); + + // Read all entities up front + ZipEntry projectsEntry = zip.getEntry("projects.json"); + try (InputStream is = zip.getInputStream(projectsEntry)) { + String json = new String(readAllBytes(is), StandardCharsets.UTF_8); + JSONArray arr = new JSONArray(json); + for (int i = 0; i < arr.length(); i++) { + OPMProject opmProject = OPMProject.fromJson(arr.getJSONObject(i)); + opmProjectsByName.put(opmProject.projectName, opmProject); + } + } + ZipEntry projectVersionsEntry = zip.getEntry("project_versions.json"); + + try (InputStream is = zip.getInputStream(projectVersionsEntry)) { + String json = new String(readAllBytes(is), StandardCharsets.UTF_8); + JSONArray arr = new JSONArray(json); + projectVersions = new ArrayList<>(); + for (int i = 0; i < arr.length(); i++) { + projectVersions.add(OPMProjectVersion.fromJson(arr.getJSONObject(i))); + } + } + ZipEntry modulesEntry = zip.getEntry("modules.json"); + ZipEntry moduleVersionsEntry = zip.getEntry("module_versions.json"); + + if (modulesEntry != null && moduleVersionsEntry != null) { + try (InputStream is = zip.getInputStream(modulesEntry)) { + String json = new String(readAllBytes(is), StandardCharsets.UTF_8); + JSONArray arr = new JSONArray(json); + + for (int i = 0; i < arr.length(); i++) { + DecisionServiceProject module = DecisionServiceProject.fromJson(arr.getJSONObject(i)); + decisionServiceProjectsByName.put(module.projectName, module); + } + } + try (InputStream is = zip.getInputStream(moduleVersionsEntry)) { + String json = new String(readAllBytes(is), StandardCharsets.UTF_8); + JSONArray arr = new JSONArray(json); + for (int i = 0; i < arr.length(); i++) { + projectVersions.add(DecisionServiceProjectVersion.fromJson(arr.getJSONObject(i))); + } + } + } + + return projectVersions; + } + + + /** + * Uploads an OPM project version with its snapshot to the IA Hub. + * @param projectVersion Project version metadata and refs. + * @param zip Payload zip containing the snapshot. + * @param iaHostUrl Base URL of the IA Hub. + * @param oAuthToken OAuth bearer token. + * @throws Exception on HTTP or serialization errors. + */ + private void importOPMProjectVersion(OPMProjectVersion projectVersion, ZipFile zip, String iaHostUrl, String oAuthToken) throws Exception { + String projectName = projectVersion.projectName; + int projectVersionNumber = projectVersion.projectVersionNumber; + String versionDescription = projectVersion.description; + String userName = projectVersion.userName; + String opaVersion = projectVersion.opaVersion; + String creationDate = projectVersion.creationDate; + + String descriptionUpdatedDate = projectVersion.descriptionUpdated; + String descriptionAuthor = projectVersion.descriptionAuthor; + + OPMProject project = opmProjectsByName.get(projectName); + String fromProjectName = project.fromProjectName; + Integer fromProjectVersionNumber = project.fromProjectVersionNumber; + String workspace = project.workspace; + + // Get snapshot fingerprint + String fingerprint = projectVersion.fingerprintSha256; + + // Read snapshot bytes from zip + ZipEntry snapshotEntry = zip.getEntry(fingerprint); + if (snapshotEntry == null) { + throw new RuntimeException("Snapshot file not found in zip: " + fingerprint); + } + + byte[] snapshotBytes; + try (InputStream is = zip.getInputStream(snapshotEntry)) { + snapshotBytes = readAllBytes(is); + } + + // project changes + Map changes = projectVersion.changes; + + // Call uploadProjectVersion with status messages + uploadProjectVersion(iaHostUrl, oAuthToken, projectName, projectVersionNumber, versionDescription, userName, opaVersion, creationDate, + workspace, descriptionUpdatedDate, descriptionAuthor, changes, fromProjectName, fromProjectVersionNumber, snapshotBytes); + } + + /** + * Uploads a policy-model project version to the IA Hub. + * @param iaHostUrl Base URL of the IA Hub. + * @param oAuthToken OAuth bearer token. + * @param projectName Project name. + * @param projectVersionNumber Version number. + * @param description Version description. + * @param userName Author of the version. + * @param opaVersion OPA version string. + * @param creationDate Creation timestamp. + * @param workspace Target workspace name. + * @param descriptionUpdatedDate When the description was last updated. + * @param descriptionAuthor Who last updated the description. + * @param changes Object-level change map. + * @param fromProjectName Source project name for v1 cloning (optional). + * @param fromProjectVersionNumber Source project version for v1 cloning (optional). + * @param snapshotBytes Snapshot payload bytes. + * @throws Exception if the HTTP request fails or Hub returns a non-2xx status. + */ + private void uploadProjectVersion(String iaHostUrl, + String oAuthToken, + String projectName, + int projectVersionNumber, + String description, + String userName, + String opaVersion, + String creationDate, + String workspace, + String descriptionUpdatedDate, + String descriptionAuthor, + Map changes, + String fromProjectName, + Integer fromProjectVersionNumber, + byte[] snapshotBytes) throws Exception { + + // Build request JSON + JSONObject body = new JSONObject(); + body.put("project_name", projectName); + body.put("project_version_number", projectVersionNumber); + body.put("opa_version", opaVersion); + body.put("description", description); + body.put("user_name", userName); + body.put("creation_date", creationDate); + body.put("workspace", workspace); + body.put("description_updated", descriptionUpdatedDate); + body.put("description_author", descriptionAuthor); + + if (projectVersionNumber == 1) { + if (fromProjectName != null) { + body.put("from_project_name", fromProjectName); + if (fromProjectVersionNumber != null) { + body.put("from_project_version_number", fromProjectVersionNumber); + } + } + } + + if (!changes.isEmpty()) { + JSONObject changesObj = new JSONObject(); + for (String objectName : changes.keySet()) { + changesObj.put(objectName, changes.get(objectName)); + } + body.put("changes", changesObj); + } + + JSONObject snapshot = new JSONObject(); + + // Encode snapshotBytes as base64 + String snapshotBase64 = Base64.getEncoder().encodeToString(snapshotBytes); + snapshot.put("snapshot_base64", snapshotBase64); + + body.put("snapshot", snapshot); + + // Build URL + String url = iaHostUrl + opmProjectUrlPath; + + Map headers = new HashMap<>(); + headers.put(HttpHeaders.AUTHORIZATION, "Bearer " + oAuthToken); + headers.put(HttpHeaders.ACCEPT, ContentType.APPLICATION_JSON.getMimeType()); + headers.put(HttpHeaders.CONTENT_TYPE, ContentType.APPLICATION_JSON.getMimeType()); + HttpResult httpRes = httpTransport.request("POST", url, headers, body.toString()); + int statusCode = httpRes.statusCode; + String responseText = httpRes.body; + + if (statusCode < 200 || statusCode >= 300) { + throw new RuntimeException("Policy Modeling project upload failed. HTTP code: " + statusCode + "\nResponse: " + responseText); + } + } + + + /** + * Uploads a decision service project version to the IA Hub and records it in the journal. + * @param moduleVersion Decision service project version metadata and content. + * @param iaHostUrl Base URL of the IA Hub. + * @param oAuthToken OAuth bearer token. + * @param journal Journal to write success entries to. + * @throws Exception if the HTTP request fails or Hub returns a non-2xx status. + */ + private void importModuleVersion(DecisionServiceProjectVersion moduleVersion, String iaHostUrl, String oAuthToken, Journal journal) throws Exception{ + String projectName = moduleVersion.projectName; + DecisionServiceProject module = decisionServiceProjectsByName.get(projectName); + + JSONObject postBody = new JSONObject(); + + if (module != null && module.fromProjectName != null) { + postBody.put("from_module_name", module.fromProjectName); + if (module.fromVersionNumber != null) { + postBody.put("from_version_number", module.fromVersionNumber); + } + } + + postBody.put("module_name", projectName); + if (module != null && module.workspace != null) { + postBody.put("workspace", module.workspace); + } + postBody.put("create_timestamp", moduleVersion.createTimestamp); + postBody.put("module_imported", moduleVersion.imported); + postBody.put("user_name", moduleVersion.userName); + postBody.put("version_number", moduleVersion.versionNumber); + postBody.put("definition", moduleVersion.definition); + if (moduleVersion.description != null) { + postBody.put("description", moduleVersion.description); + } + if (moduleVersion.descriptionUpdated != null) { + postBody.put("description_updated", moduleVersion.descriptionUpdated); + } + if (moduleVersion.descriptionAuthor != null) { + postBody.put("description_author", moduleVersion.descriptionAuthor); + } + postBody.put("fingerprint_sha256", moduleVersion.fingerprintSha256); + + String url = iaHostUrl + decisionServiceProjectUrlPath; + + Map headers = new HashMap<>(); + headers.put(HttpHeaders.AUTHORIZATION, "Bearer " + oAuthToken); + headers.put(HttpHeaders.ACCEPT, ContentType.APPLICATION_JSON.getMimeType()); + headers.put(HttpHeaders.CONTENT_TYPE, ContentType.APPLICATION_JSON.getMimeType()); + HttpResult httpRes = httpTransport.request("POST", url, headers, postBody.toString()); + int statusCode = httpRes.statusCode; + String responseText = httpRes.body; + + if (statusCode < 200 || statusCode >= 300) { + throw new RuntimeException("Decision service project version import failed. HTTP code: " + statusCode + "\nResponse: " + responseText); + } + } + + /** + * Obtains an OAuth access token from the IA Hub. + * @param iaHostUrl Base URL of the IA Hub. + * @param iaUsername Client ID/username. + * @param iaPassword Client secret/password. + * @return bearer access token. + * @throws RuntimeException if authentication fails or response is invalid. + */ + private String authenticate(String iaHostUrl, String iaUsername, String iaPassword) { + String authUrl = iaHostUrl + authUrlPath; + + StringBuilder form = new StringBuilder(); + form.append("grant_type=client_credentials"); + form.append("&client_id=").append(urlEncodeUtf8(iaUsername)); + form.append("&client_secret=").append(urlEncodeUtf8(iaPassword)); + + Map headers = new HashMap<>(); + headers.put(HttpHeaders.ACCEPT, ContentType.APPLICATION_JSON.getMimeType()); + headers.put(HttpHeaders.CONTENT_TYPE, ContentType.APPLICATION_FORM_URLENCODED.getMimeType()); + + HttpResult httpRes; + try { + httpRes = httpTransport.request("POST", authUrl, headers, form.toString()); + } catch (Exception ex) { + throw new RuntimeException("Exception during OAuth authentication", ex); + } + + int statusCode = httpRes.statusCode; + String responseText = httpRes.body; + + if (statusCode >= 200 && statusCode < 300) { + JSONObject tokenJson = new JSONObject(responseText); + String accessToken = tokenJson.optString("access_token", null); + if (accessToken != null && !accessToken.isEmpty()) { + return accessToken; + } else { + throw new RuntimeException("Authentication succeeded but access_token not found"); + } + } else { + throw new RuntimeException("Authentication failed. HTTP code: " + statusCode + "\nResponse: " + responseText); + } + } + + /** + * Reads an import journal and returns its entries (excluding the header). + * @param journalPath Path to the journal file. + * @return list of JSON entries after the header. + * @throws RuntimeException if the file is invalid or unreadable. + */ + private List parseJournalEntries(String journalPath) { + List journalEntries = new ArrayList<>(); + + try (BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(journalPath), StandardCharsets.UTF_8))) { + String line; + int lineNo = 0; + while ((line = br.readLine()) != null) { + + JSONObject obj; + try { + obj = new JSONObject(line); + } catch (Exception ex) { + throw new RuntimeException("Invalid journal file"); + } + + if (lineNo == 0) { + if (!obj.has("payload_sha256") || !(obj.has("hub_url"))) { + throw new RuntimeException("Invalid journal file"); + } + + resumePayloadSha256 = obj.getString("payload_sha256"); + resumeHubUrl = obj.getString("hub_url"); + } else { + journalEntries.add(obj); + } + + lineNo++; + } + } catch (IOException ex) { + throw new RuntimeException("Error reading journal file"); + } + + return journalEntries; + } + + /** + * Fetches available workspace names from the IA Hub. + * @param iaHostUrl Base URL of the IA Hub. + * @param oAuthToken OAuth bearer token. + * @return set of workspace names. + * @throws Exception if the HTTP request fails or parsing fails. + */ + private Set fetchWorkspaceNames(String iaHostUrl, String oAuthToken) throws Exception { + Set workspaceNames = new HashSet<>(); + + Map headers = new HashMap<>(); + headers.put(HttpHeaders.AUTHORIZATION, "Bearer " + oAuthToken); + headers.put(HttpHeaders.ACCEPT, ContentType.APPLICATION_JSON.getMimeType()); + headers.put(HttpHeaders.ACCEPT_ENCODING, "gzip"); + + String url = iaHostUrl + workspacesUrlPath; + + HttpResult res = httpTransport.request("GET", url, headers, null); + if (res.statusCode < 200 || res.statusCode >= 300) { + throw new RuntimeException("Failed to fetch Hub projects for resume validation. HTTP code: " + res.statusCode + "\nResponse: " + res.body); + } + + JSONObject root = new JSONObject(res.body); + if (root.has("items")) { + JSONArray items = root.getJSONArray("items"); + for (int i = 0; i < items.length(); i++) { + JSONObject workspace = items.getJSONObject(i); + String name = workspace.getString("name"); + workspaceNames.add(name); + } + } + + return workspaceNames; + } + + // Fetch projects (policy-model) and decision service projects (decision) from Hub, with versions in order + /** + * Fetches existing projects and decision service projects and their versions from the IA Hub. + * @param iaHostUrl Base URL of the IA Hub. + * @param oAuthToken OAuth bearer token. + * @return map of project name to ordered versions. + * @throws Exception if the HTTP request fails or parsing fails. + */ + private Map> fetchHubVersions(String iaHostUrl, String oAuthToken) throws Exception { + String url = iaHostUrl + projectVersionsUrlPath; + + Map headers = new HashMap<>(); + headers.put(HttpHeaders.AUTHORIZATION, "Bearer " + oAuthToken); + headers.put(HttpHeaders.ACCEPT, ContentType.APPLICATION_JSON.getMimeType()); + headers.put(HttpHeaders.ACCEPT_ENCODING, "gzip"); + + HttpResult res = httpTransport.request("GET", url, headers, null); + if (res.statusCode < 200 || res.statusCode >= 300) { + throw new RuntimeException("Failed to fetch Hub projects for resume validation. HTTP code: " + res.statusCode + "\nResponse: " + res.body); + } + + Map> projectsByName = new HashMap<>(); + + JSONObject root = new JSONObject(res.body); + if (root.has("items")) { + JSONArray items = root.getJSONArray("items"); + for (int i = 0; i < items.length(); i++) { + JSONObject proj = items.getJSONObject(i); + String name = proj.optString("name", null); + String kind = proj.optString("kind", ""); + + // Normalize kind to our importer terminology + boolean isPolicyModel = "policy-model".equalsIgnoreCase(kind); + boolean isDecision = "decision".equalsIgnoreCase(kind); + + JSONObject versionsObj = proj.optJSONObject("versions"); + if (versionsObj == null || !versionsObj.has("items")) continue; + JSONArray versionsArr = versionsObj.getJSONArray("items"); + + for (int v = 0; v < versionsArr.length(); v++) { + JSONObject ver = versionsArr.getJSONObject(v); + int versionNo = ver.getInt("version"); + String description = ver.optString("description", null); + String descriptionUpdatedAt = ver.optString("descriptionUpdatedAt", null); + String descriptionAuthor = ver.optString("descriptionAuthor", null); + String author = ver.optString("author", null); + String createTimestamp = ver.optString("createTimestamp", null); + String fingerprintSha256 = ver.optString("definitionHash", null); + + if (isPolicyModel) { + // Map to ProjectVersion (fields not present on Hub remain null) + OPMProjectVersion pv = new OPMProjectVersion( + name, + versionNo, + description, + author, + null, // opaVersion not provided by Projects API + createTimestamp, + descriptionUpdatedAt, + descriptionAuthor, + fingerprintSha256, + null + ); + projectsByName.computeIfAbsent(name, k -> new ArrayList<>()).add(pv); + } else if (isDecision) { + // Map to ModuleVersion + String definition = ver.has("definition") ? ver.get("definition").toString() : null; + boolean isDraft = ver.getBoolean("isDraft"); + DecisionServiceProjectVersion mv = new DecisionServiceProjectVersion( + name, + isDraft ? 0 : versionNo, + createTimestamp, + 0, // moduleImported not on Projects API; set default + author, + definition, + description, + descriptionUpdatedAt, + descriptionAuthor, + fingerprintSha256 + ); + projectsByName.computeIfAbsent(name, k -> new ArrayList<>()).add(mv); + } + } + } + } + + for (List projectVersions : projectsByName.values()) { + projectVersions.sort(Comparator.comparing(ProjectVersion::isDraft).thenComparingInt(ProjectVersion::getVersion)); + } + + return projectsByName; + } + + // Generic HTTP utility for GET/POST requests + /** + * Simple HTTP result holder for status and body. + */ + public static class HttpResult { + final int statusCode; + final String body; + HttpResult(int statusCode, String body) { + this.statusCode = statusCode; + this.body = body != null ? body : ""; + } + } + + /** + * Default HTTP transport using Apache HttpClient. + */ + private static class DefaultHttpTransport implements HttpTransport { + /** + * Delegates to the static httpRequest helper. + */ + @Override + public HttpResult request(String method, String url, Map headers, String body) throws Exception { + return httpRequest(method, url, headers, body); + } + } + + /** + * Performs a GET or POST request and returns status/body. + * @param method HTTP method to use. + * @param url Target URL. + * @param headers Request headers to include. + * @param body Optional UTF-8 body for POST. + * @return HTTP result containing status and body text. + * @throws Exception if the HTTP client fails or the request cannot be executed. + */ + private static HttpResult httpRequest(String method, String url, Map headers, String body) throws Exception { + try (CloseableHttpClient httpClient = HttpClients.createDefault()) { + if ("GET".equalsIgnoreCase(method)) { + HttpGet request = new HttpGet(url); + if (headers != null) { + for (Map.Entry h : headers.entrySet()) { + request.setHeader(h.getKey(), h.getValue()); + } + } + try (CloseableHttpResponse response = httpClient.execute(request)) { + int sc = response.getStatusLine().getStatusCode(); + HttpEntity entity = response.getEntity(); + String responseText = entity != null ? EntityUtils.toString(entity, "UTF-8") : ""; + return new HttpResult(sc, responseText); + } + } else if ("POST".equalsIgnoreCase(method)) { + HttpPost request = new HttpPost(url); + if (headers != null) { + for (Map.Entry h : headers.entrySet()) { + request.setHeader(h.getKey(), h.getValue()); + } + } + if (body != null) { + request.setEntity(new StringEntity(body, "UTF-8")); + } + try (CloseableHttpResponse response = httpClient.execute(request)) { + int sc = response.getStatusLine().getStatusCode(); + HttpEntity entity = response.getEntity(); + String responseText = entity != null ? EntityUtils.toString(entity, "UTF-8") : ""; + return new HttpResult(sc, responseText); + } + } else { + throw new IllegalArgumentException("Unsupported HTTP method: " + method); + } + } + } + + /** + * URL-encodes a string as UTF-8. + * @param s String to encode. + * @return encoded string. + */ + private static String urlEncodeUtf8(String s) { + try { + return URLEncoder.encode(s, "UTF-8"); + } catch (java.io.UnsupportedEncodingException e) { + throw new RuntimeException(e); + } + } + + // Helper method to read all bytes from an InputStream (replacement for StreamUtils.readAll) + /** + * Reads all bytes from the given input stream. + * @param input Input stream to read. + * @return byte array of all read data. + * @throws IOException if an I/O error occurs. + */ + private static byte[] readAllBytes(InputStream input) throws IOException { + ByteArrayOutputStream buffer = new ByteArrayOutputStream(); + int nRead; + byte[] data = new byte[4096]; + while ((nRead = input.read(data, 0, data.length)) != -1) { + buffer.write(data, 0, nRead); + } + buffer.flush(); + return buffer.toByteArray(); + } +} diff --git a/tools/project-history-migrator-tool/src/main/java/com/oracle/determinations/migration/Journal.java b/tools/project-history-migrator-tool/src/main/java/com/oracle/determinations/migration/Journal.java new file mode 100644 index 0000000..405d264 --- /dev/null +++ b/tools/project-history-migrator-tool/src/main/java/com/oracle/determinations/migration/Journal.java @@ -0,0 +1,64 @@ +package com.oracle.determinations.migration; + +import org.json.JSONObject; + +import java.io.BufferedWriter; +import java.io.FileOutputStream; +import java.io.OutputStreamWriter; +import java.nio.charset.StandardCharsets; + +/** + * Append-only UTF-8 journal that writes one JSON object (or string) per line. + * Ensures data is flushed and synced to disk to minimize loss on failure. + */ +public class Journal implements AutoCloseable { + private final FileOutputStream fos; + private final BufferedWriter writer; + + /** + * Creates a journal appending to the given file path. + * @param path file path to write journal entries to. + * @throws java.io.IOException if the file cannot be opened for append. + */ + public Journal(String path) throws java.io.IOException { + this.fos = new FileOutputStream(path, true); + this.writer = new BufferedWriter(new OutputStreamWriter(fos, StandardCharsets.UTF_8)); + } + + /** + * Writes a JSON object as a single line entry. + * @param obj JSON object to write. + * @throws java.io.IOException if an I/O error occurs. + */ + public synchronized void write(JSONObject obj) throws java.io.IOException { + write(obj.toString()); + } + + /** + * Writes a raw string as a single line entry and fsyncs the file. + * @param str line to write (a newline will be appended). + * @throws java.io.IOException if an I/O error occurs. + */ + public synchronized void write(String str) throws java.io.IOException { + writer.write(str); + writer.newLine(); + writer.flush(); + // Force data to disk to minimize loss if process terminates unexpectedly + this.fos.getFD().sync(); + } + + /** + * Flushes, fsyncs, and closes the underlying streams. + * @throws java.io.IOException if closing or syncing fails. + */ + @Override + public void close() throws java.io.IOException { + try { + writer.flush(); + this.fos.getFD().sync(); + } finally { + writer.close(); + fos.close(); + } + } +} diff --git a/tools/project-history-migrator-tool/src/main/java/com/oracle/determinations/migration/Main.java b/tools/project-history-migrator-tool/src/main/java/com/oracle/determinations/migration/Main.java new file mode 100644 index 0000000..43f9d9c --- /dev/null +++ b/tools/project-history-migrator-tool/src/main/java/com/oracle/determinations/migration/Main.java @@ -0,0 +1,182 @@ +package com.oracle.determinations.migration; + +import java.io.BufferedOutputStream; +import java.io.FileOutputStream; +import java.io.Console; +import java.io.BufferedReader; +import java.io.InputStreamReader; +import java.io.IOException; +import java.util.zip.ZipOutputStream; +import java.time.ZoneId; +import java.time.DateTimeException; + +/** + * CLI entry point for exporting from a database or importing into IA Hub. + */ +public class Main { + /** + * Application entry point. + * @param args Command-line arguments. Use: + * --export [timeZoneId] or --import [resumeJournalPath] + */ + public static void main(String[] args) { + try { + if (args.length < 1) { + printUsage(); + return; + } + + String mode = args[0].toLowerCase(); + + switch (mode) { + case "--export": + // Expect: --export [timeZoneId]; db credentials read from stdin + if (args.length != 2 && args.length != 3) { + printUsage(); + return; + } + String dbUrl = args[1]; + ZoneId exportZoneId = null; + if (args.length == 3) { + try { + exportZoneId = ZoneId.of(args[2]); + } catch (DateTimeException dte) { + System.err.println("Invalid time zone ID: " + args[2] + ". See ZoneId IDs like 'UTC', 'Europe/London', 'America/New_York', '+10:00'."); + System.exit(1); + } + } + String username = prompt("Database username: "); + String password = promptPassword("Database password: "); + + System.out.println("Export Mode Selected."); + System.out.println("Database URL: " + dbUrl); + System.out.println("Database username: " + username); + if (exportZoneId != null) { + System.out.println("Export time zone: " + exportZoneId); + } else { + System.out.println("Export time zone: (system default) " + ZoneId.systemDefault()); + } + + try { + Exporter exporter = (exportZoneId != null) ? new Exporter(exportZoneId) : new Exporter(); + String exportZip = exporter.newExportZipFileName(); + try (java.sql.Connection conn = exporter.establishConnection(dbUrl, username, password)) { + try ( + FileOutputStream fos = new FileOutputStream(exportZip); + BufferedOutputStream bos = new BufferedOutputStream(fos); + ZipOutputStream zos = new ZipOutputStream(bos) + ) { + exporter.doExport(conn, zos); + zos.flush(); + System.out.println("Exported data to " + exportZip); + } + } + } catch (ClassNotFoundException e) { + System.err.println("JDBC Driver not found for URL: " + dbUrl + ". Ensure the appropriate MySQL or Oracle JDBC driver is on the classpath."); + System.exit(1); + } catch (java.sql.SQLException e) { + System.err.println("Failed to connect to the database: " + e.getMessage()); + System.exit(1); + } catch (Exception e) { + System.err.println("Export failed: " + e.getMessage()); + System.exit(1); + } + break; + case "--import": + // Expect: --import [resumeJournalPath] + // API client identifier and secret read from stdin + if (args.length != 3 && args.length != 4) { + printUsage(); + return; + } + String iaHostUrl = args[1]; + if (iaHostUrl.endsWith("/")) { + iaHostUrl = iaHostUrl.substring(0, iaHostUrl.length() - 1); + } + + String exportedPayloadPath = args[2]; + String resumeJournalPath = args.length == 4 ? args[3] : null; + + String iaClientId = prompt("API client identifier: "); + String iaClientSecret = promptPassword("API client secret: "); + + System.out.println("Import Mode Selected."); + System.out.println("Intelligent Advisor Host URL: " + iaHostUrl); + System.out.println("API client identifier: " + iaClientId); + System.out.println("API client secret: (provided, hidden for security)"); + System.out.println("Exported Data Payload Path: " + exportedPayloadPath); + if (resumeJournalPath != null) { + System.out.println("Resume Journal: " + resumeJournalPath); + } + + try { + new Importer().doImport(iaHostUrl, iaClientId, iaClientSecret, exportedPayloadPath, resumeJournalPath); + } catch (Exception e) { + System.err.println("Import failed: " + e.getMessage()); + System.exit(1); + } + break; + default: + printUsage(); + break; + } + } catch (Exception e) { + System.err.println("Error: " + e.getMessage()); + System.exit(1); + } + + } + + /** + * Prompts the user for input and returns the trimmed response. + * @param prompt Message to display. + * @return trimmed input line (empty string if null). + * @throws RuntimeException if reading from stdin fails. + */ + private static String prompt(String prompt) { + Console console = System.console(); + if (console != null) { + String line = console.readLine("%s", prompt); + return line != null ? line.trim() : ""; + } + try { + System.out.print(prompt); + BufferedReader br = new BufferedReader(new InputStreamReader(System.in)); + String line = br.readLine(); + return line != null ? line.trim() : ""; + } catch (IOException e) { + throw new RuntimeException("Failed to read input from stdin", e); + } + } + + /** + * Prompts the user for a password (masked when a console is available). + * Falls back to visible input if no console is available. + * @param prompt Message to display. + * @return entered password string (may be empty). + */ + private static String promptPassword(String prompt) { + Console console = System.console(); + if (console != null) { + char[] pwd = console.readPassword("%s", prompt); + return pwd != null ? new String(pwd) : ""; + } + // Fallback (visible) when no console is available, e.g., running in some IDEs + return prompt(prompt); + } + + /** + * Prints usage information for the CLI. + */ + private static void printUsage() { + System.out.println("Usage:"); + System.out.println(" java -jar project-history-migrator-tool.jar --export [timeZoneId]"); + System.out.println(" - Optional timeZoneId sets the time zone for interpreting and formatting DB timestamps (e.g., 'UTC', 'Europe/London', '+10:00'). Defaults to system time zone. See Java 8 Javadoc ZoneId.of(String): https://docs.oracle.com/javase/8/docs/api/java/time/ZoneId.html#of-java.lang.String-"); + System.out.println(" - Prompts for the database username and password via stdin."); + System.out.println(); + System.out.println(" java -jar project-history-migrator-tool.jar --import [resumeJournalPath]"); + System.out.println(" - Prompts for the API client identifier and secret via stdin."); + System.out.println(" - Imports the exported.zip payload into the given Intelligent Advisor server."); + System.out.println(" - If resumeJournalPath is provided, validates journal entries against the payload and resumes by skipping versions already imported."); + } +} diff --git a/tools/project-history-migrator-tool/src/main/java/com/oracle/determinations/migration/OPMProject.java b/tools/project-history-migrator-tool/src/main/java/com/oracle/determinations/migration/OPMProject.java new file mode 100644 index 0000000..e851ca3 --- /dev/null +++ b/tools/project-history-migrator-tool/src/main/java/com/oracle/determinations/migration/OPMProject.java @@ -0,0 +1,41 @@ +package com.oracle.determinations.migration; + +import org.json.JSONObject; + +/** + * Represents an OPM policy model project, including optional source project linkage and workspace. + */ +public class OPMProject { + public final String projectName; + public final String fromProjectName; + public final Integer fromProjectVersionNumber; + public final String workspace; + + /** + * Constructs an OPMProject descriptor. + * @param projectName Project name. + * @param workspace Workspace the project belongs to. + * @param fromProjectName Optional source project name for initial version cloning. + * @param fromProjectVersionNumber Optional source project version number for cloning. + */ + public OPMProject(String projectName, String workspace, String fromProjectName, Integer fromProjectVersionNumber) { + this.projectName = projectName; + this.workspace = workspace; + this.fromProjectName = fromProjectName; + this.fromProjectVersionNumber = fromProjectVersionNumber; + } + + /** + * Builds an OPMProject from a JSON object. + * @param obj JSON with keys: project_name, workspace, and optional from_project_name, from_project_version_number. + * @return parsed OPMProject instance. + * @throws org.json.JSONException if required fields are missing or invalid. + */ + public static OPMProject fromJson(JSONObject obj) { + String name = obj.getString("project_name"); + String workspace = obj.getString("workspace"); + String fromName = obj.has("from_project_name") ? obj.getString("from_project_name") : null; + Integer fromVersion = obj.has("from_project_version_number") ? obj.getInt("from_project_version_number") : null; + return new OPMProject(name, workspace, fromName, fromVersion); + } +} diff --git a/tools/project-history-migrator-tool/src/main/java/com/oracle/determinations/migration/OPMProjectVersion.java b/tools/project-history-migrator-tool/src/main/java/com/oracle/determinations/migration/OPMProjectVersion.java new file mode 100644 index 0000000..550eb19 --- /dev/null +++ b/tools/project-history-migrator-tool/src/main/java/com/oracle/determinations/migration/OPMProjectVersion.java @@ -0,0 +1,159 @@ +package com.oracle.determinations.migration; + +import org.json.JSONArray; +import org.json.JSONObject; + +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; + +/** + * Represents an OPM project version with metadata, inclusion overrides, and changes. + */ +public class OPMProjectVersion implements ProjectVersion { + public final String projectName; + public final int projectVersionNumber; + public final String description; + public final String userName; + public final String opaVersion; + public final String creationDate; + public final String descriptionUpdated; + public final String descriptionAuthor; + public final String fingerprintSha256; + public final Map changes; + + /** + * Constructs an OPMProjectVersion. + * @param projectName Project name. + * @param projectVersionNumber Version number. + * @param description Version description. + * @param userName Author of the version. + * @param opaVersion OPA version string. + * @param creationDate Creation timestamp. + * @param descriptionUpdated When the description was last updated. + * @param descriptionAuthor Who last updated the description. + * @param fingerprintSha256 Snapshot fingerprint hash. + * @param changes Object-level change map for the version. + */ + public OPMProjectVersion(String projectName, + int projectVersionNumber, + String description, + String userName, + String opaVersion, + String creationDate, + String descriptionUpdated, + String descriptionAuthor, + String fingerprintSha256, + Map changes) { + this.projectName = projectName; + this.projectVersionNumber = projectVersionNumber; + this.description = description; + this.userName = userName; + this.opaVersion = opaVersion; + this.creationDate = creationDate; + this.descriptionUpdated = descriptionUpdated; + this.descriptionAuthor = descriptionAuthor; + this.fingerprintSha256 = fingerprintSha256; + this.changes = changes != null ? changes : new HashMap<>(); + } + + /** + * Builds an OPMProjectVersion from JSON. + * @param obj JSON with keys: project_name, project_version_number, description, user_name, opa_version, creation_date, optional description_updated, description_author, fingerprint_sha256, project_version_inclusions, project_version_changes. + * @return parsed OPMProjectVersion. + * @throws org.json.JSONException if required fields are missing or invalid. + */ + public static OPMProjectVersion fromJson(JSONObject obj) { + String projectName = obj.getString("project_name"); + int versionNumber = obj.getInt("project_version_number"); + String description = obj.getString("description"); + String userName = obj.getString("user_name"); + String opaVersion = obj.getString("opa_version"); + String creationDate = obj.getString("creation_date"); + String descriptionUpdated = obj.has("description_updated") ? obj.getString("description_updated") : null; + String descriptionAuthor = obj.has("description_author") ? obj.getString("description_author") : null; + String fingerprint = obj.getString("fingerprint_sha256"); + + Map changes = new HashMap<>(); + if (obj.has("project_version_changes")) { + JSONArray chArr = obj.getJSONArray("project_version_changes"); + for (int j = 0; j < chArr.length(); j++) { + JSONObject ch = chArr.getJSONObject(j); + changes.put(ch.getString("object_name"), ch.getString("change_type")); + } + } + + return new OPMProjectVersion(projectName, versionNumber, description, userName, opaVersion, creationDate, + descriptionUpdated, descriptionAuthor, fingerprint, changes); + } + + + /** + * Serializes fields needed for the import journal. + * @param index sequential index for the journal entry. + * @return JSON entry for the journal. + */ + public JSONObject toJSONForJournal(int index) { + JSONObject obj = new JSONObject(); + obj.put("index", index); + obj.put("project_name", projectName); + obj.put("version_number", projectVersionNumber); + obj.put("type", "policy-model"); + obj.put("sha256", fingerprintSha256); + return obj; + } + + /** + * Value equality based on key fields. + */ + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof OPMProjectVersion)) return false; + OPMProjectVersion that = (OPMProjectVersion) o; + return projectVersionNumber == that.projectVersionNumber + && Objects.equals(projectName, that.projectName) + && Objects.equals(description, that.description) + && Objects.equals(userName, that.userName) + && DateTimeUtil.sameInstant(creationDate, that.creationDate) + && DateTimeUtil.sameInstant(descriptionUpdated, that.descriptionUpdated) + && Objects.equals(descriptionAuthor, that.descriptionAuthor) + && Objects.equals(fingerprintSha256, that.fingerprintSha256); + } + + /** + * Hash code consistent with equals. + */ + @Override + public int hashCode() { + return Objects.hash(projectName, projectVersionNumber, description, userName, + DateTimeUtil.instantOrNull(creationDate), + DateTimeUtil.instantOrNull(descriptionUpdated), + descriptionAuthor, fingerprintSha256); + } + + /** + * Returns the version number. + */ + @Override + public int getVersion() { + return projectVersionNumber; + } + + /** + * Returns the project name. + */ + @Override + public String getProjectName() { + return projectName; + } + + /** + * OPM projects do not support draft versions. + */ + @Override + public boolean isDraft() { + // No drafts for OPM projects + return false; + } +} diff --git a/tools/project-history-migrator-tool/src/main/java/com/oracle/determinations/migration/ProjectVersion.java b/tools/project-history-migrator-tool/src/main/java/com/oracle/determinations/migration/ProjectVersion.java new file mode 100644 index 0000000..5a85cf8 --- /dev/null +++ b/tools/project-history-migrator-tool/src/main/java/com/oracle/determinations/migration/ProjectVersion.java @@ -0,0 +1,21 @@ +package com.oracle.determinations.migration; + +/** + * Common metadata for a project version. + */ +public interface ProjectVersion { + + /** + * Returns the version number. + */ + int getVersion(); + /** + * Returns the project name. + */ + String getProjectName(); + + /** + * True if this version is a draft (unreleased). + */ + boolean isDraft(); +} diff --git a/tools/project-history-migrator-tool/src/test/java/com/oracle/determinations/migration/ImportTest.java b/tools/project-history-migrator-tool/src/test/java/com/oracle/determinations/migration/ImportTest.java new file mode 100644 index 0000000..ca5e559 --- /dev/null +++ b/tools/project-history-migrator-tool/src/test/java/com/oracle/determinations/migration/ImportTest.java @@ -0,0 +1,647 @@ +package com.oracle.determinations.migration; + +import org.json.JSONArray; +import org.json.JSONObject; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileWriter; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.util.*; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; + +import static org.junit.Assert.*; + +/** + * Unit tests for the Importer, covering happy path, validation, auth, and resume logic. + */ +public class ImportTest { + + @Rule + public TemporaryFolder tmp = new TemporaryFolder(); + + // Test helper: records requests and replays queued responses + /** + * Fake HTTP transport for tests: records requests and returns queued responses. + */ + static class FakeHttpTransport implements Importer.HttpTransport { + /** + * Captured HTTP request details for assertions. + */ + static class Request { + final String method, url, body; + final Map headers; + /** + * Constructs a captured request. + * @param method HTTP method. + * @param url Target URL. + * @param headers Request headers (copied). + * @param body Request body string (may be null). + */ + Request(String method, String url, Map headers, String body) { + this.method = method; + this.url = url; + this.headers = headers != null ? new HashMap<>(headers) : new HashMap<>(); + this.body = body; + } + } + /** + * Pre-canned HTTP response to be dequeued by the fake transport. + */ + static class Response { + final int status; + final String body; + /** + * Constructs a fake response. + * @param status HTTP status code. + * @param body Response body. + */ + Response(int status, String body) { + this.status = status; + this.body = body; + } + } + + final List requests = new ArrayList<>(); + final Deque responses = new ArrayDeque<>(); + + /** + * Enqueues a fake response to return for the next request. + * @param status HTTP status code. + * @param body Body text to return. + */ + void addResponse(int status, String body) { + responses.addLast(new Response(status, body)); + } + + /** + * Returns all captured requests in order. + * @return list of recorded requests. + */ + List getRequests() { + return requests; + } + + @Override + /** + * Records the request and returns the next queued response or a default 200 {}. + * @param method HTTP method. + * @param url Target URL. + * @param headers Headers to send. + * @param body Optional body for POST. + * @return result containing status code and body. + */ + public Importer.HttpResult request(String method, String url, Map headers, String body) { + requests.add(new Request(method, url, headers, body)); + Response r = responses.isEmpty() ? new Response(200, "{}") : responses.removeFirst(); + return new Importer.HttpResult(r.status, r.body); + } + } + + // Override to control journal filename deterministically + /** + * Importer variant that writes to a deterministic journal path for tests. + */ + static class TestImporter extends Importer { + private final String journalPath; + /** + * Creates a test importer with injected seams and fixed journal path. + * @param httpTransport fake HTTP transport. + * @param journalFactory journal factory. + * @param journalPath output path for the journal file. + */ + public TestImporter(HttpTransport httpTransport, JournalFactory journalFactory, String journalPath) { + super(httpTransport, journalFactory); + this.journalPath = journalPath; + } + @Override + /** + * Uses a predictable journal file name in tests. + * @return configured journal path. + */ + protected String newJournalFileName() { + return journalPath; + } + } + + // Utilities + + /** + * Builds a zip with text and binary entries. + * @param name zip filename. + * @param entries map of entryName -> text content (UTF-8). + * @param binaryEntries map of entryName -> raw bytes. + * @return the created zip File. + * @throws Exception on I/O error. + */ + private File buildZip(String name, Map entries, Map binaryEntries) throws Exception { + File zip = tmp.newFile(name); + try (ZipOutputStream zos = new ZipOutputStream(Files.newOutputStream(zip.toPath()))) { + if (entries != null) { + for (Map.Entry e : entries.entrySet()) { + zos.putNextEntry(new ZipEntry(e.getKey())); + byte[] bytes = e.getValue().getBytes(StandardCharsets.UTF_8); + zos.write(bytes); + zos.closeEntry(); + } + } + if (binaryEntries != null) { + for (Map.Entry e : binaryEntries.entrySet()) { + zos.putNextEntry(new ZipEntry(e.getKey())); + zos.write(e.getValue()); + zos.closeEntry(); + } + } + } + return zip; + } + + /** + * Creates a projects API response body with provided items array. + * @param items array of project items. + * @return JSON string. + */ + private String hubProjectsResponse(JSONArray items) { + JSONObject root = new JSONObject(); + root.put("items", items); + return root.toString(); + } + + /** + * Builds a project item with versions. + * @param name project/module name. + * @param kind policy-model or decision. + * @param versionItems versions array. + * @return JSON object. + */ + private JSONObject hubProjectItem(String name, String kind, JSONArray versionItems) { + JSONObject proj = new JSONObject(); + proj.put("name", name); + proj.put("kind", kind); + JSONObject versions = new JSONObject(); + versions.put("items", versionItems); + proj.put("versions", versions); + return proj; + } + + /** + * Builds a policy-model version object for the Hub response. + * @param version version number. + * @param author author name. + * @param description version description. + * @param createTimestamp timestamp string. + * @param defHash definition hash. + * @return JSON version object. + */ + private JSONObject hubPolicyModelVersion(int version, String author, String description, String createTimestamp, String defHash) { + JSONObject v = new JSONObject(); + v.put("version", version); + v.put("author", author); + v.put("description", description); + v.put("createTimestamp", createTimestamp); + v.put("definitionHash", defHash); + // optional descriptionUpdatedAt, descriptionAuthor omitted + return v; + } + + /** + * Builds a decision version object for the Hub response. + * @param version version number. + * @param isDraft whether this version is a draft. + * @param author author name. + * @param description version description. + * @param createTimestamp timestamp string. + * @param defHash definition hash. + * @return JSON version object. + */ + private JSONObject hubDecisionVersion(int version, boolean isDraft, String author, String description, String createTimestamp, String defHash) { + JSONObject v = new JSONObject(); + v.put("version", version); + v.put("isDraft", isDraft); + v.put("author", author); + v.put("description", description); + v.put("createTimestamp", createTimestamp); + v.put("definitionHash", defHash); + // optional definition omitted + return v; + } + + /** + * Creates a workspaces API response body with given workspace names. + * @param names workspace names. + * @return JSON string. + */ + private String workspacesResponse(String... names) { + JSONArray items = new JSONArray(); + for (String n : names) { + JSONObject obj = new JSONObject(); + obj.put("name", n); + items.put(obj); + } + JSONObject root = new JSONObject(); + root.put("items", items); + return root.toString(); + } + + /** + * Builds minimal payload entries for a mixed policy-model and decision module. + * @param projectName name of the policy-model project. + * @param workspace1 workspace for the project. + * @param moduleName name of the decision module. + * @param workspace2 workspace for the module. + * @param fingerprint snapshot fingerprint entry name. + * @return map of entry name to JSON string. + */ + private Map payloadEntriesForMixed(String projectName, String workspace1, String moduleName, String workspace2, String fingerprint) { + // projects.json + JSONArray projects = new JSONArray(); + JSONObject p = new JSONObject(); + p.put("project_name", projectName); + p.put("workspace", workspace1); + projects.put(p); + + // project_versions.json + JSONArray projectVersions = new JSONArray(); + JSONObject pv = new JSONObject(); + pv.put("project_name", projectName); + pv.put("project_version_number", 1); + pv.put("description", "desc"); + pv.put("user_name", "user1"); + pv.put("opa_version", "12.2.39"); + pv.put("creation_date", "2024-01-01T00:00:00Z"); + pv.put("fingerprint_sha256", fingerprint); + projectVersions.put(pv); + + // modules.json + JSONArray modules = new JSONArray(); + JSONObject m = new JSONObject(); + m.put("module_name", moduleName); + m.put("workspace", workspace2); + modules.put(m); + + // module_versions.json + JSONArray moduleVersions = new JSONArray(); + JSONObject mv = new JSONObject(); + mv.put("module_name", moduleName); + mv.put("version_number", 1); + mv.put("create_timestamp", "2024-02-01T00:00:00Z"); + mv.put("module_imported", 0); + mv.put("user_name", "user2"); + mv.put("definition", "{\"dsl\":\"ok\"}"); + mv.put("fingerprint_sha256", "defhash"); + moduleVersions.put(mv); + + Map jsons = new HashMap<>(); + jsons.put("projects.json", projects.toString()); + jsons.put("project_versions.json", projectVersions.toString()); + jsons.put("modules.json", modules.toString()); + jsons.put("module_versions.json", moduleVersions.toString()); + return jsons; + } + + @Test + /** + * Verifies a mixed payload imports successfully and writes expected journal entries. + * @throws Exception on unexpected failure. + */ + public void testImportsMixedPayloadSuccessfully() throws Exception { + String projectName = "PolicyA"; + String moduleName = "DecisionA"; + String ws1 = "WS1"; + String ws2 = "WS2"; + String fingerprint = "snap1"; + + Map entries = payloadEntriesForMixed(projectName, ws1, moduleName, ws2, fingerprint); + Map bin = new HashMap<>(); + bin.put(fingerprint, "SNAPSHOT".getBytes(StandardCharsets.UTF_8)); + File zip = buildZip("payload.zip", entries, bin); + + // Prepare fake HTTP + FakeHttpTransport http = new FakeHttpTransport(); + // 1) auth + http.addResponse(200, new JSONObject().put("access_token", "token").toString()); + // 2) fetchHubVersions (empty) + http.addResponse(200, hubProjectsResponse(new JSONArray())); + // 3) fetch workspaces + http.addResponse(200, workspacesResponse(ws1, ws2)); + // 4) POST OPM + http.addResponse(201, "{}"); + // 5) POST Module + http.addResponse(201, "{}"); + + File journal = tmp.newFile("journal.json"); + // clean it so Importer appends to empty + Files.write(journal.toPath(), new byte[0]); + + Importer imp = new TestImporter(http, Journal::new, journal.getAbsolutePath()); + imp.doImport("https://host", "user", "pass", zip.getAbsolutePath(), null); + + // Assert 5 requests made + List reqs = http.getRequests(); + assertEquals(5, reqs.size()); + assertEquals("POST", reqs.get(0).method); // auth + assertTrue(reqs.get(1).url.contains("/projects?expand=versions")); + assertTrue(reqs.get(2).url.contains("/workspaces")); + // The two uploads + FakeHttpTransport.Request post1 = reqs.get(3); + FakeHttpTransport.Request post2 = reqs.get(4); + assertEquals("POST", post1.method); + assertEquals("POST", post2.method); + assertTrue(post1.url.contains("/opa-hub/api/experimental/migrate-opm-project-version")); + assertTrue(post2.url.contains("/opa-hub/api/experimental/migrate-decision-service-project-version")); + + // Validate posted JSON bodies + JSONObject body1 = new JSONObject(post1.body); + assertEquals(projectName, body1.getString("project_name")); + assertEquals(1, body1.getInt("project_version_number")); + assertEquals(ws1, body1.getString("workspace")); + assertTrue(body1.getJSONObject("snapshot").has("snapshot_base64")); + + JSONObject body2 = new JSONObject(post2.body); + assertEquals(moduleName, body2.getString("module_name")); + assertEquals(1, body2.getInt("version_number")); + assertEquals(ws2, body2.getString("workspace")); + assertEquals("{\"dsl\":\"ok\"}", body2.getString("definition")); + assertFalse(body2.has("description")); // optional absent by our input + + // Validate journal: header + 2 entries + List lines = Files.readAllLines(journal.toPath(), StandardCharsets.UTF_8); + assertEquals(3, lines.size()); + JSONObject header = new JSONObject(lines.get(0)); + assertEquals("https://host", header.getString("hub_url")); + JSONObject e0 = new JSONObject(lines.get(1)); + JSONObject e1 = new JSONObject(lines.get(2)); + assertEquals(0, e0.getInt("index")); + assertEquals(1, e1.getInt("index")); + } + + @Test + /** + * Fails when project/module names already exist on the Hub. + * @throws Exception on unexpected failure. + */ + public void testFailsOnProjectNameClashes() throws Exception { + String projectName = "ExistingProject"; + String moduleName = "ExistingModule"; + String ws = "WS1"; + String fingerprint = "fp"; + + Map entries = payloadEntriesForMixed(projectName, ws, moduleName, ws, fingerprint); + Map bin = new HashMap<>(); + bin.put(fingerprint, "SNAP".getBytes(StandardCharsets.UTF_8)); + File zip = buildZip("payload.zip", entries, bin); + + FakeHttpTransport http = new FakeHttpTransport(); + // auth ok + http.addResponse(200, new JSONObject().put("access_token", "t").toString()); + // fetchHubVersions returns names that will clash (must include at least one version) + JSONArray items = new JSONArray(); + items.put(hubProjectItem(projectName, "policy-model", new JSONArray().put(hubPolicyModelVersion(1, "a", "d", "t", "h")))); + items.put(hubProjectItem(moduleName, "decision", new JSONArray().put(hubDecisionVersion(1, false, "a", "d", "t", "h")))); + http.addResponse(200, hubProjectsResponse(items)); + // workspaces won't be reached due to clash check, but provide anyway + http.addResponse(200, workspacesResponse(ws)); + + Importer imp = new TestImporter(http, Journal::new, tmp.newFile("j.json").getAbsolutePath()); + + try { + imp.doImport("https://host", "u", "p", zip.getAbsolutePath(), null); + fail("Expected RuntimeException for name clashes"); + } catch (RuntimeException ex) { + assertTrue(ex.getMessage().contains("already exist on the IA Hub")); + assertTrue(ex.getMessage().contains(projectName)); + assertTrue(ex.getMessage().contains(moduleName)); + } + } + + @Test + /** + * Fails when required workspaces are not present on the Hub. + * @throws Exception on unexpected failure. + */ + public void testFailsOnMissingWorkspaces() throws Exception { + String projectName = "P1"; + String moduleName = "M1"; + String fingerprint = "f1"; + + Map entries = payloadEntriesForMixed(projectName, "WS_REQ_1", moduleName, "WS_REQ_2", fingerprint); + Map bin = new HashMap<>(); + bin.put(fingerprint, "X".getBytes(StandardCharsets.UTF_8)); + File zip = buildZip("payload.zip", entries, bin); + + FakeHttpTransport http = new FakeHttpTransport(); + // auth ok + http.addResponse(200, new JSONObject().put("access_token", "t").toString()); + // no existing projects + http.addResponse(200, hubProjectsResponse(new JSONArray())); + // workspaces response missing required ones + http.addResponse(200, workspacesResponse("WS_OTHER")); + + Importer imp = new TestImporter(http, Journal::new, tmp.newFile("j.json").getAbsolutePath()); + + try { + imp.doImport("https://host", "u", "p", zip.getAbsolutePath(), null); + fail("Expected RuntimeException for missing workspaces"); + } catch (RuntimeException ex) { + assertTrue(ex.getMessage().contains("Target hub is missing workspaces")); + assertTrue(ex.getMessage().contains("WS_REQ_1")); + assertTrue(ex.getMessage().contains("WS_REQ_2")); + } + } + + @Test + /** + * Resume should fail if journal payload hash does not match the payload. + * @throws Exception on unexpected failure. + */ + public void testResumeJournalPayloadShaMismatch() throws Exception { + // Minimal payload: one project with fingerprint + String fingerprint = "f1"; + Map entries = payloadEntriesForMixed("P1", "WS", "M1", "WS", fingerprint); + Map bin = new HashMap<>(); + bin.put(fingerprint, "X".getBytes(StandardCharsets.UTF_8)); + File zip = buildZip("payload.zip", entries, bin); + + // Create a resume journal with mismatching sha + File resume = tmp.newFile("resume.json"); + JSONObject header = new JSONObject(); + header.put("payload_sha256", "NOT_MATCHING"); + header.put("hub_url", "https://host"); + Files.write(resume.toPath(), (header.toString() + "\n").getBytes(StandardCharsets.UTF_8)); + + FakeHttpTransport http = new FakeHttpTransport(); + // auth ok + http.addResponse(200, new JSONObject().put("access_token", "t").toString()); + // fetchHubVersions ok (won't reach mismatch until after this call) + http.addResponse(200, hubProjectsResponse(new JSONArray())); + + Importer imp = new TestImporter(http, Journal::new, tmp.newFile("j.json").getAbsolutePath()); + + try { + imp.doImport("https://host", "u", "p", zip.getAbsolutePath(), resume.getAbsolutePath()); + fail("Expected RuntimeException for journal payload mismatch"); + } catch (RuntimeException ex) { + assertTrue(ex.getMessage().contains("Journal file does not match payload")); + } + } + + @Test + /** + * Resume should fail if journal hub URL does not match the target IA host. + * @throws Exception on unexpected failure. + */ + public void testResumeJournalHubUrlMismatch() throws Exception { + String fingerprint = "f2"; + Map entries = payloadEntriesForMixed("P1", "WS", "M1", "WS", fingerprint); + Map bin = new HashMap<>(); + bin.put(fingerprint, "X".getBytes(StandardCharsets.UTF_8)); + File zip = buildZip("payload.zip", entries, bin); + + File resume = tmp.newFile("resume.json"); + JSONObject header = new JSONObject(); + header.put("payload_sha256", sha256OfFile(zip)); + header.put("hub_url", "https://DIFFERENT"); + Files.write(resume.toPath(), (header.toString() + "\n").getBytes(StandardCharsets.UTF_8)); + + FakeHttpTransport http = new FakeHttpTransport(); + http.addResponse(200, new JSONObject().put("access_token", "t").toString()); + http.addResponse(200, hubProjectsResponse(new JSONArray())); + + Importer imp = new TestImporter(http, Journal::new, tmp.newFile("j.json").getAbsolutePath()); + try { + imp.doImport("https://host", "u", "p", zip.getAbsolutePath(), resume.getAbsolutePath()); + fail("Expected RuntimeException for hub url mismatch"); + } catch (RuntimeException ex) { + assertTrue(ex.getMessage().contains("Journal file does not match specified IA Hub")); + } + } + + @Test + /** + * Auth 200 response without access_token should be treated as an error. + * @throws Exception on unexpected failure. + */ + public void testAuthSuccessMissingAccessToken() throws Exception { + String fingerprint = "f3"; + Map entries = payloadEntriesForMixed("P1", "WS", "M1", "WS", fingerprint); + Map bin = new HashMap<>(); + bin.put(fingerprint, "X".getBytes(StandardCharsets.UTF_8)); + File zip = buildZip("payload.zip", entries, bin); + + FakeHttpTransport http = new FakeHttpTransport(); + // auth returns 200 without access_token + http.addResponse(200, "{}"); + + Importer imp = new TestImporter(http, Journal::new, tmp.newFile("j.json").getAbsolutePath()); + try { + imp.doImport("https://host", "u", "p", zip.getAbsolutePath(), null); + fail("Expected RuntimeException for missing access_token"); + } catch (RuntimeException ex) { + assertTrue(ex.getMessage().contains("Authentication succeeded but access_token not found")); + } + } + + @Test + /** + * Non-2xx auth responses should cause an authentication error. + * @throws Exception on unexpected failure. + */ + public void testAuthNon2xxFailure() throws Exception { + String fingerprint = "f4"; + Map entries = payloadEntriesForMixed("P1", "WS", "M1", "WS", fingerprint); + Map bin = new HashMap<>(); + bin.put(fingerprint, "X".getBytes(StandardCharsets.UTF_8)); + File zip = buildZip("payload.zip", entries, bin); + + FakeHttpTransport http = new FakeHttpTransport(); + // auth fails + http.addResponse(401, "{\"error\":\"bad creds\"}"); + + Importer imp = new TestImporter(http, Journal::new, tmp.newFile("j.json").getAbsolutePath()); + try { + imp.doImport("https://host", "u", "p", zip.getAbsolutePath(), null); + fail("Expected RuntimeException for auth failure"); + } catch (RuntimeException ex) { + assertTrue(ex.getMessage().contains("Authentication failed")); + assertTrue(ex.getMessage().contains("401")); + } + } + + @Test + /** + * Resuming should replay prior entries and continue with remaining items. + * @throws Exception on unexpected failure. + */ + public void testResumeReplaysEntriesAndContinues() throws Exception { + String projectName = "P1"; + String moduleName = "M1"; + String ws = "WS"; + String fingerprint = "f5"; + + Map entries = payloadEntriesForMixed(projectName, ws, moduleName, ws, fingerprint); + Map bin = new HashMap<>(); + bin.put(fingerprint, "X".getBytes(StandardCharsets.UTF_8)); + File zip = buildZip("payload.zip", entries, bin); + + // Resume with one prior entry (index 0) + File resume = tmp.newFile("resume.json"); + String header = new JSONObject().put("payload_sha256", sha256OfFile(zip)).put("hub_url", "https://host").toString(); + String priorEntry = new JSONObject().put("index", 0).put("project_name", projectName).put("type", "policy-model").put("version_number", 1).toString(); + Files.write(resume.toPath(), (header + "\n" + priorEntry + "\n").getBytes(StandardCharsets.UTF_8)); + + FakeHttpTransport http = new FakeHttpTransport(); + // auth ok + http.addResponse(200, new JSONObject().put("access_token", "t").toString()); + // fetchHubVersions returns that P1 v1 exists so resume validation passes (match payload fields for equals()) + JSONArray items = new JSONArray(); + items.put(hubProjectItem(projectName, "policy-model", new JSONArray().put(hubPolicyModelVersion(1, "user1", "desc", "t", "f5")))); + http.addResponse(200, hubProjectsResponse(items)); + // Only remaining import is module M1 + http.addResponse(201, "{}"); // POST module + + File journalOut = tmp.newFile("journal-resume.json"); + Importer imp = new TestImporter(http, Journal::new, journalOut.getAbsolutePath()); + + imp.doImport("https://host", "u", "p", zip.getAbsolutePath(), resume.getAbsolutePath()); + + // Requests: POST auth, GET projects, POST module (workspaces not called in resume path) + List reqs = http.getRequests(); + assertEquals(3, reqs.size()); + assertEquals("POST", reqs.get(0).method); + assertTrue(reqs.get(1).url.contains("/projects?expand=versions")); + assertEquals("POST", reqs.get(2).method); + assertTrue(reqs.get(2).url.contains("/opa-hub/api/experimental/migrate-decision-service-project-version")); // module uses same path per current code + + // Journal should contain header + prior entry + new entry + List lines = Files.readAllLines(journalOut.toPath(), StandardCharsets.UTF_8); + assertEquals(3, lines.size()); + JSONObject newEntry = new JSONObject(lines.get(2)); + assertEquals(1, newEntry.getInt("index")); + assertEquals("decision", newEntry.getString("type")); + assertEquals(moduleName, newEntry.getString("project_name")); + } + + // Helper: compute sha256 of a file via Java (matches main code semantics) + /** + * Computes SHA-256 of a file for tests. + * @param file target file. + * @return lowercase hex digest. + * @throws Exception if reading the file or digesting fails. + */ + private String sha256OfFile(File file) throws Exception { + byte[] bytes = Files.readAllBytes(file.toPath()); + // Lightweight inline SHA-256 to avoid adding dependencies in test; not strictly needed for speed + java.security.MessageDigest md = java.security.MessageDigest.getInstance("SHA-256"); + byte[] digest = md.digest(bytes); + StringBuilder sb = new StringBuilder(); + for (byte b : digest) { + sb.append(String.format("%02x", b)); + } + return sb.toString(); + } +}