Handle gsi

Dockerfile

@@ -0,0 +1,5 @@
+FROM node:7.9.0-alpine
+
+# Add project
+COPY . /home/root
+RUN cd /home/root && npm install

README.md

@@ -25,6 +25,7 @@ Can be run as a command line script or as an npm module.
     -p, --backup-path <name>         backup path to store table dumps in. default is DynamoDB-backup-YYYY-MM-DD-HH-mm-ss
     -e, --base64-encode-binary       if passed, encode binary fields in base64 before exporting
     -d, --save-datapipeline-format   save in format compatible with the AWS datapipeline import. Default to false (save as exported by DynamoDb)
+    -f, --save-schema                save table schema. Default to true
     --aws-key                        AWS access key. Will use AWS_ACCESS_KEY_ID env var if --aws-key not set
     --aws-secret                     AWS secret key. Will use AWS_SECRET_ACCESS_KEY env var if --aws-secret not set
     --aws-region                     AWS region. Will use AWS_DEFAULT_REGION env var if --aws-region not set
@@ -161,7 +162,7 @@ __Arguments__
 
 It is suitable for restoring large tables without needing to write to disk or use a large amount of memory. Use it on an AWS EC2 instance for best results and to minimise network latency, this should yield restore speeds of around 15min per GB.
 
-Use `--overwrite` if the table already exists. Otherwise it will attempt to generate table on the fly.
+Use `--overwrite` if the table already exists. Otherwise it will attempt to generate table on the fly using table.schema.json created on the backup.
 
 Can be run as a command line script or as an npm module. 
 

backup.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+./home/root/bin/dynamo-backup-to-s3 --bucket uniplaces.com.backups -p $(date +%Y-%m-%d) -r 0.1 --aws-key $AWS_KEY_BACKUP --aws-secret $AWS_SECRET_BACKUP --aws-region eu-west-1 --excluded-tables prod-search-offers,prod-admin-session,prod-ap-session,prod-core-session,prod-ops-session,prod-photography-session,prod-spa-session

bin/dynamo-backup-to-s3

@@ -26,6 +26,7 @@ program
     .option('-p, --backup-path <name>', 'backup path to store table dumps in. default is DynamoDB-backup-YYYY-MM-DD-HH-mm-ss')
     .option('-e, --base64-encode-binary', 'encode binary fields in base64 before exporting')
     .option('-d, --save-data-pipeline-format', 'save in format compatible with the AWS Data Pipeline import. Default to false (save as exported by DynamoDb)')
+    .option('-f, --save-schema', 'save table schema. Default to false')
     .option('--aws-key <key>', 'AWS access key. Will use AWS_ACCESS_KEY_ID env var if --aws-key not set')
     .option('--aws-secret <secret>', 'AWS secret key. Will use AWS_SECRET_ACCESS_KEY env var if --aws-secret not set')
     .option('--aws-region <region>', 'AWS region. Will use AWS_DEFAULT_REGION env var if --aws-region not set')
@@ -46,7 +47,8 @@ var dynamoBackup = new DynamoBackup({
     readPercentage: program.readPercentage,
     stopOnFailure: program.stopOnFailure,
     base64Binary: program.base64EncodeBinary,
-    saveDataPipelineFormat: program.saveDataPipelineFormat
+    saveDataPipelineFormat: program.saveDataPipelineFormat,
+    saveSchema: program.saveSchema
 });
 
 dynamoBackup.on('error', function(data) {

bin/dynamo-restore-from-s3

@@ -1,5 +1,6 @@
 #!/usr/bin/env node
-
+var MY_SLACK_WEBHOOK_URL = 'https://hooks.slack.com/services/T1JC01J20/B56R0998F/' + process.env.SLACK_ID;
+var slack = require('slack-notify')(MY_SLACK_WEBHOOK_URL);
 var program = require('commander'),
     fs = require('fs'),
     DynamoRestore = require('../').Restore;
@@ -11,6 +12,7 @@ program
     .option('-t, --table [name]', 'Name of the Dynamo Table to restore to (Required)')
     .option('-o, --overwrite', 'Table already exists, skip auto-create. Default is false.')
     .option('-c, --concurrency <requestcount>', 'Number of concurrent requests & dynamo capacity units. Defaults to 200.')
+    .option('-ic, --index-concurrency <requestcount>', 'Number of concurrent requests & dynamo capacity units for global secondary indexes. Defaults to 200.')
     .option('-pk, --partitionkey [columnname]', 'Name of Primary Partition Key. If not provided will try determine from backup.')
     .option('-sk, --sortkey [columnname]', 'Name of Secondary Sort Key. Ignored unless --partitionkey is provided.')
     .option('-rc, --readcapacity <units>', 'Read Units for new table (when finished). Default is 5.')
@@ -39,6 +41,7 @@ var dynamoRestore = new DynamoRestore({
     table: program.table,
     overwrite: !!program.overwrite,
     concurrency: program.concurrency,
+    indexConcurrency: program.indexConcurrency,
     stopOnFailure: !!program.stopOnFailure,
     // New table properties
     partitionkey: program.partitionkey,
@@ -62,20 +65,34 @@ function translate(contentLength) {
 
 // Define events
 dynamoRestore.on('error', function(message) {
+    slack.note({
+        channel: '#ped_backup_restore',
+        text: 'Something went wrong restoring: ' + program.table,
+        fields: {
+            'Error': message,
+        },
+        username: 'Pato'
+    });
     console.log(message);
-    process.exit(-1);
+    setTimeout(function() { process.exit(-1); }, 3000);
 });
 
 dynamoRestore.on('warning', function(message) {
     console.log(message);
 });
 
 dynamoRestore.on('start-download', function(streamMeta) {
+    slack.note({
+        channel: '#ped_backup_restore',
+        text: 'Restoring ' + program.table + '. Write capacity set to: ' +  program.concurrency,
+        username: 'Pato'
+    });
     var time = runTimes.startDownload = new Date().getTime();
     console.log('Starting download. %s remaining...', translate(streamMeta.ContentLength));
 });
 
 dynamoRestore.on('send-batch', function(batches, requests, streamMeta) {
+    return;
     console.log('Batch sent. %d in flight. %s remaining to download...', requests, translate(streamMeta.RemainingLength));
 });
 
@@ -85,6 +102,12 @@ dynamoRestore.run(function() {
         diff = time - runTimes.start,
         minutes = Math.floor(diff / (1000 * 60)),
         seconds = Math.floor((diff % 1000 * 60) / 1000);
+
+    slack.note({
+        channel: '#ped_backup_restore',
+        text: 'Restore completed for ' + program.table + ' in ' + minutes + ' minutes ' + seconds + ' seconds. Write capacity: ' +  program.writecapacity + ', Read capacity: ' + program.writecapacity,
+        username: 'Pato'
+    });
     console.log('Done! Process completed in %s minutes %s seconds.', minutes, seconds);
-    process.exit(0);
-});
+    setTimeout(function() { process.exit(0); }, 3000);
+});

deploy.sh

@@ -0,0 +1,67 @@
+# SET YOUR AWS CONFIG BEFORE RUNNING
+$(aws ecr get-login --region eu-west-1)
+docker build -t dynamo-backup-to-s3 .
+docker tag dynamo-backup-to-s3:latest 584629324139.dkr.ecr.eu-west-1.amazonaws.com/dynamo-backup-to-s3:latest
+docker push 584629324139.dkr.ecr.eu-west-1.amazonaws.com/dynamo-backup-to-s3:latest
+
+# more bash-friendly output for jq
+JQ="jq --raw-output --exit-status"
+
+# create task def
+make_task_def() {
+	task_def="[
+		{
+			\"name\": \"dynamo-backup-to-s3\",
+			\"image\": \"584629324139.dkr.ecr.eu-west-1.amazonaws.com/dynamo-backup-to-s3:latest\",
+			\"essential\": true,
+			\"memory\": 2500,
+			\"cpu\": 800,
+            \"logConfiguration\": {
+                \"logDriver\": \"awslogs\",
+                \"options\": {
+                    \"awslogs-group\": \"prod-jobs\",
+                    \"awslogs-region\": \"eu-west-1\",
+                    \"awslogs-stream-prefix\": \"dynamo-backup-job\"
+                }
+		    },
+            \"environment\": [
+                {
+                    \"name\": \"AWS_KEY_BACKUP\",
+                    \"value\":\"$AWS_KEY_BACKUP\"
+                },
+                {
+                    \"name\": \"AWS_SECRET_BACKUP\",
+                    \"value\": \"$AWS_SECRET_BACKUP\"
+                },
+                {
+                    \"name\": \"AWS_KEY_RESTORE\",
+                    \"value\": \"$AWS_KEY_RESTORE\"
+                },
+                {
+                    \"name\": \"AWS_SECRET_RESTORE\",
+                    \"value\": \"$AWS_SECRET_RESTORE\"
+                },
+                {
+                    \"name\": \"SLACK_ID\",
+                    \"value\": \"$SLACK_ID\"
+                }
+            ]
+        }
+	]"
+}
+
+# register definition
+register_definition() {
+    family="dynamo-backup-to-s3"
+    if revision=$(aws ecs register-task-definition --container-definitions "$task_def" --family $family | $JQ '.taskDefinition.taskDefinitionArn'); then
+        echo "Revision: $revision"
+        echo "$revision" > arn_revision.txt
+        echo $revision | sed -n -e 's/^.*task-definition\///p' > task_revision.txt
+    else
+        echo "Failed to register task definition"
+        return 1
+    fi
+}
+
+make_task_def
+register_definition

ecs-overrides-backup.json

@@ -0,0 +1,8 @@
+{
+    "containerOverrides": [
+        {
+            "name": "dynamo-backup-to-s3",
+            "command": ["sh","/home/root/backup.sh"]
+        }
+    ]
+}

ecs-overrides-restore.json

@@ -0,0 +1,8 @@
+{
+    "containerOverrides": [
+        {
+            "name": "dynamo-backup-to-s3",
+            "command": ["sh","/home/root/restore.sh"]
+        }
+    ]
+}