CLDSRV-717: Test configured as S3C Integration

.github/actions/setup-ci/action.yaml

@@ -9,6 +9,7 @@ runs:
       shell: bash
       run: |
         sudo echo "127.0.0.1 bucketwebsitetester.s3-website-us-east-1.amazonaws.com" | sudo tee -a /etc/hosts
+        sudo echo "127.0.0.1 bucketwebsitetester.s3-website-us-east-1.scality.com" | sudo tee -a /etc/hosts
         sudo echo "127.0.0.1 pykmip.local" | sudo tee -a /etc/hosts
         for i in `seq 1 50`; do sudo echo "127.0.0.$i $i.pykmip.local" | sudo tee -a /etc/hosts ; done
     - name: Setup Credentials

.github/docker/config.s3c.json

@@ -0,0 +1,63 @@
+{
+    "port": 8000,
+    "maxScannedLifecycleListingEntries": 10000,
+    "metricsPort": 8001,
+    "internalPort": 8004,
+    "replicationGroupId": "RG001",
+    "restEndpoints": {
+        "s3.amazonaws.com": "dc-1",
+        "localhost": "dc-1",
+        "scality.scality.com": "dc-1",
+        "127.0.0.1": "dc-1",
+        "node1.scality.com": "dc-1"
+    },
+    "websiteEndpoints": ["s3-website-us-east-1.scality.com"],
+    "bucketd": {
+        "bootstrap": ["0.0.0.0:9000"]
+    },
+    "vaultd": {
+        "host": "127.0.0.1",
+        "port": 8500
+    },
+    "clusters": 10,
+    "kmsHideScalityArn": true,
+    "healthChecks": {
+        "allowFrom": ["127.0.0.1", "::1"]
+    },
+    "localCache": {
+        "host": "localhost",
+        "port": 6479,
+        "password": ""
+    },
+    "log": {
+        "logLevel": "info",
+        "dumpLevel": "error"
+    },
+    "replicationEndpoints": [
+        {"site": "zenko", "servers": ["127.0.0.1:9080"], "echo": false, "default": true}, 
+        {"site": "us-east-2", "servers": ["127.0.0.1:9080"]}
+    ],
+    "requests": {
+        "extractClientIPFromHeader": "x-forwarded-for",
+        "extractProtocolFromHeader": "x-forwarded-proto",
+        "trustedProxyCIDRs": [
+            "127.0.0.1/32",
+            "::ffff:127.0.0.1/128",
+            "127.0.0.1"
+        ],
+        "viaProxy": true
+    },
+    "multiObjectDeleteEnableOptimizations": false,
+    "supportedLifecycleRules": [
+        "Expiration",
+        "NoncurrentVersionExpiration",
+        "AbortIncompleteMultipartUpload"
+    ],
+    "bucketNotificationDestinations": [
+        {
+            "resource": "target1",
+            "type": "dummy",
+            "host": "localhost:6000"
+        }
+    ]
+}

.github/docker/docker-compose.sse.yaml

@@ -36,7 +36,15 @@ services:
       - ../../tests/functional/sse-kms-migration/config.json:/conf/config.json
     environment:
       - S3_CONFIG_FILE=/conf/config.json
+      - S3_LOCATION_FILE=/usr/src/app/tests/locationConfig/locationConfigS3C.json
   cloudserver-sse-migration:
     extends: cloudserver-sse-before-migration
     profiles: [sse-migration]
     command: sh -c "yarn start > /artifacts/s3.migration.log 2> /artifacts/s3-stderr.migration.log"
+  metadata-standalone:
+    image: ghcr.io/scality/metadata:8.11.0-standalone
+    profiles: ['metadata-standalone']
+    network_mode: 'host'
+    volumes:
+      - ./md-config-v0.json:/mnt/standalone_workdir/config.json:ro
+      - /tmp/artifacts/${JOB_NAME}/md:/mnt/standalone_workdir/log

.github/docker/md-config-v0.json

@@ -0,0 +1,29 @@
+{
+    "raftSessions": 1,
+    "raftMembers": 5,
+    "bucketdCount": 1,
+    "bucketdWorkers": 1,
+    "basePorts": {
+        "bucketd": 9000,
+        "repd": 4200,
+        "repdAdmin": 4250
+    },
+    "logLevel": "info",
+    "env": {
+        "METADATA_NEW_BUCKETS_VFORMAT": "v0",
+        "S3_VERSION_ID_ENCODING_TYPE":"hex"
+    },
+    "migration": {
+        "deploy": false,
+        "raftSessions": 0,
+        "raftMembers": 5,
+        "bucketdCount": 1,
+        "bucketdWorkers": 1,
+        "basePorts": {
+            "bucketd": 9001,
+            "repd": 4700,
+            "repdAdmin": 4750
+        },
+        "logLevel": "info"
+    }
+}

.github/docker/setup-s3c.sh

@@ -0,0 +1,74 @@
+#!/bin/bash
+
+# setup S3C environment just like
+# https://github.com/scality/Integration/blob/development/9.5/tests/setup-environment/index.js
+
+# credentials:
+# default = account1
+# lisa = account2
+# see https://github.com/scality/Integration/blob/development/9.5/tests/cloudserver/runTests.js
+
+set -e -o pipefail
+
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+
+VAULTCLIENT=$SCRIPT_DIR/../../node_modules/vaultclient/bin/vaultclient
+CONFIG=$SCRIPT_DIR/admin.json
+
+echo "Deleting and setting up S3C accounts like Integration (follow conf/authdata.json)"
+
+$VAULTCLIENT --config $CONFIG delete-account --name Bart || true
+$VAULTCLIENT --config $CONFIG create-account \
+    --name Bart \
+    --email [email protected] \
+    --accountid 123456789012 \
+    --canonicalid 79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be
+$VAULTCLIENT --config $CONFIG generate-account-access-key \
+    --name Bart \
+    --accesskey ACC1AK00000000000000 \
+    --secretkey ACC1SK0000000000000000000000000000000000
+
+$VAULTCLIENT --config $CONFIG delete-account --name Lisa || true
+$VAULTCLIENT --config $CONFIG create-account \
+    --name Lisa \
+    --email [email protected] \
+    --accountid 123456789013 \
+    --canonicalid 79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2bf
+$VAULTCLIENT --config $CONFIG generate-account-access-key \
+    --name Lisa \
+    --accesskey ACC2AK00000000000000 \
+    --secretkey ACC2SK0000000000000000000000000000000000
+
+# Replication account for backbeat replication tests
+$VAULTCLIENT --config $CONFIG delete-account --name Replication || true
+# Cannot use url as canonicalid for service account
+$VAULTCLIENT --config $CONFIG create-account \
+    --name Replication \
+    --email [email protected] \
+    --accountid 123456789015 \
+    --canonicalid 79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2ba
+$VAULTCLIENT --config $CONFIG generate-account-access-key \
+    --name Replication \
+    --accesskey ACCREPAK000000000000 \
+    --secretkey ACCREPSK00000000000000000000000000000000
+
+echo "Copying s3c credentials to mem credentials"
+cp \
+    $SCRIPT_DIR/../../tests/functional/aws-node-sdk/lib/json/s3c_credentials.json \
+    $SCRIPT_DIR/../../tests/functional/aws-node-sdk/lib/json/mem_credentials.json
+
+echo "Update conf/authdata.json account Replication canonicalID"
+REP_CANONICAL_ID=$(
+    $VAULTCLIENT --config $CONFIG get-account --account-name Replication | jq -r .canonicalId
+)
+
+# jq reformats the whole json file so use sed instead
+# jq \
+#     "(.accounts[] | select(.name == \"Replication\") | .canonicalID) |= \"$REP_CANONICAL_ID\"" \
+#     $SCRIPT_DIR/../../conf/authdata.json \
+#     > $SCRIPT_DIR/../../conf/authdata.json.tmp \
+#     && mv $SCRIPT_DIR/../../conf/authdata.json.tmp $SCRIPT_DIR/../../conf/authdata.json
+
+# might need to undo changes if script was already ran before
+git checkout -- $SCRIPT_DIR/../../conf/authdata.json
+sed -i "s/http:\/\/acs.zenko.io\/accounts\/service\/replication/$REP_CANONICAL_ID/g" $SCRIPT_DIR/../../conf/authdata.json