Datafold demo feature

.github/workflows/dbt_deploy.yml

@@ -15,23 +15,26 @@ jobs:
 
       - uses: actions/setup-python@v2
         with:
-          python-version: "3.11"
+          python-version: "3.8"
 
       - name: install requirements
         run: pip install -q -r requirements.txt
 
+      - name: install datafold-sdk
+        run: pip install -q datafold-sdk
+
       - name: dbt deps
         run: dbt deps
 
       # TODO: update for your unique S3 bucket name
       # TODO: update your GitHub secrets to include AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY
       - name: Grab production manifest from S3
         run: |
-          aws s3 cp s3://vhol-datafold-dbt-prod-manifest/manifest.json ./manifest.json
+          aws s3 cp s3://vhold-prod-manifest-demo-saravula/manifest.json ./manifest.json
         env:
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-          AWS_REGION: us-west-2
+          AWS_REGION: us-east-1
 
       - name: dbt build
         run: dbt build --target prod --select state:modified+ --defer --state ./ --exclude config.materialized:snapshot --full-refresh --profiles-dir ./
@@ -46,9 +49,9 @@ jobs:
       - name: submit artifacts to datafold
         run: |
           set -ex
-          datafold dbt upload --ci-config-id 345 --run-type ${DATAFOLD_RUN_TYPE} --commit-sha ${GIT_SHA}
+          datafold dbt upload --ci-config-id 399 --run-type ${DATAFOLD_RUN_TYPE} --commit-sha ${GIT_SHA}
         env: # TODO: update your GitHub secrets to include DATAFOLD_APIKEY
-          DATAFOLD_APIKEY: ${{ secrets.DATAFOLD_APIKEY }}
+          DATAFOLD_APIKEY: ${{ secrets.DATAFOLD_API_KEY }}
           DATAFOLD_RUN_TYPE: "${{ 'production' }}"
           GIT_SHA:
             "${{ github.sha }}"
@@ -57,8 +60,8 @@ jobs:
       # TODO: update your GitHub secrets to include AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY
       - name: Upload manifest to S3
         run: |
-          aws s3 cp target/manifest.json s3://vhol-datafold-dbt-prod-manifest/manifest.json
+          aws s3 cp target/manifest.json s3://vhold-prod-manifest-demo-saravula/manifest.json
         env:
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-          AWS_REGION: us-west-2
+          AWS_REGION: us-east-1

.github/workflows/dbt_staging.yml

@@ -40,7 +40,7 @@ jobs:
       # TODO: update your GitHub secrets to include AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY
       - name: Grab production manifest from S3
         run: |
-          aws s3 cp s3://vhol-datafold-dbt-prod-manifest/manifest.json ./manifest.json
+          aws s3 cp s3://vhold-prod-manifest-demo-saravula/manifest.json ./manifest.json
         env:
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
@@ -60,7 +60,7 @@ jobs:
       - name: submit artifacts to datafold
         run: |
           set -ex
-          datafold dbt upload --ci-config-id 345 --run-type ${DATAFOLD_RUN_TYPE} --commit-sha ${GIT_SHA}
+          datafold dbt upload --ci-config-id 400 --run-type ${DATAFOLD_RUN_TYPE} --commit-sha ${GIT_SHA}
         env: # TODO: update your GitHub secrets to include DATAFOLD_APIKEY
           DATAFOLD_APIKEY: ${{ secrets.DATAFOLD_APIKEY }}
           DATAFOLD_RUN_TYPE: "${{ 'pull_request' }}"

.gitignore

@@ -162,4 +162,6 @@ cython_debug/
 
 target/
 dbt_packages/
-logs/
+logs/
+dbt-env/bin/python
+dbt-env/bin/python3

dbt_project.yml

@@ -27,9 +27,9 @@ clean-targets:         # directories to be removed by `dbt clean`
 # TODO: update the following to your Snowflake account and Datafold Snowflake datasource_id ID
 vars:
   data_diff:
-    prod_database: DEMO
-    prod_schema: CORE
-    datasource_id: 4932
+    prod_database: DATAFOLD_DEMO
+    prod_schema: PROD
+    datasource_id: 7980
 
 # Configuring models
 # Full documentation: https://docs.getdbt.com/docs/configuring-models

models/core/dim_orgs.sql

@@ -1,3 +1,4 @@
+#
 WITH orgs AS (
 --prod
     SELECT

profiles.yml

@@ -7,19 +7,20 @@ vhol_demo:
       account: "{{ env_var('SNOWFLAKE_ACCOUNT') }}"
       user: "{{ env_var('SNOWFLAKE_USER') | as_text }}"
       password: "{{ env_var('SNOWFLAKE_PASSWORD') | as_text }}"
-      role: DEMO_ROLE
-      database: DEMO
-      warehouse: INTEGRATION
-      schema: "{{ env_var('SNOWFLAKE_SCHEMA') | as_text }}"
+      role: DATAFOLD_ROLE
+      database: DATAFOLD_DEMO
+      warehouse: TRANSFORMING
+     # schema: "{{ env_var('SNOWFLAKE_SCHEMA') | as_text }}"
+      schema: TEST_DATAFOLD_TMP 
       threads: 24
 
     prod:
       type: snowflake
       account: "{{ env_var('SNOWFLAKE_ACCOUNT') }}"
       user: "{{ env_var('SNOWFLAKE_USER') | as_text }}"
       password: "{{ env_var('SNOWFLAKE_PASSWORD') | as_text }}"
-      role: DEMO_ROLE
-      database: DEMO
-      warehouse: INTEGRATION
-      schema: CORE
-      threads: 24
+      role: DATAFOLD_ROLE
+      database: DATAFOLD_DEMO
+      warehouse: TRANSFORMING
+      schema: PROD
+      threads: 24

requirements.txt

@@ -1,4 +1,4 @@
-dbt-core==1.7.7
-dbt-snowflake==1.7.1
-data-diff==0.11.0
-datafold-sdk==0.0.19
+dbt-core
+dbt-snowflake
+data-diff
+datafold-sdk