diff --git a/.github/workflows/dbt_deploy.yml b/.github/workflows/dbt_deploy.yml index 0e8bead..c4dc30a 100644 --- a/.github/workflows/dbt_deploy.yml +++ b/.github/workflows/dbt_deploy.yml @@ -15,11 +15,14 @@ jobs: - uses: actions/setup-python@v2 with: - python-version: "3.11" + python-version: "3.8" - name: install requirements run: pip install -q -r requirements.txt + - name: install datafold-sdk + run: pip install -q datafold-sdk + - name: dbt deps run: dbt deps @@ -27,11 +30,11 @@ jobs: # TODO: update your GitHub secrets to include AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY - name: Grab production manifest from S3 run: | - aws s3 cp s3://vhol-datafold-dbt-prod-manifest/manifest.json ./manifest.json + aws s3 cp s3://vhold-prod-manifest-demo-saravula/manifest.json ./manifest.json env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_REGION: us-west-2 + AWS_REGION: us-east-1 - name: dbt build run: dbt build --target prod --select state:modified+ --defer --state ./ --exclude config.materialized:snapshot --full-refresh --profiles-dir ./ @@ -46,9 +49,9 @@ jobs: - name: submit artifacts to datafold run: | set -ex - datafold dbt upload --ci-config-id 345 --run-type ${DATAFOLD_RUN_TYPE} --commit-sha ${GIT_SHA} + datafold dbt upload --ci-config-id 399 --run-type ${DATAFOLD_RUN_TYPE} --commit-sha ${GIT_SHA} env: # TODO: update your GitHub secrets to include DATAFOLD_APIKEY - DATAFOLD_APIKEY: ${{ secrets.DATAFOLD_APIKEY }} + DATAFOLD_APIKEY: ${{ secrets.DATAFOLD_API_KEY }} DATAFOLD_RUN_TYPE: "${{ 'production' }}" GIT_SHA: "${{ github.sha }}" @@ -57,8 +60,8 @@ jobs: # TODO: update your GitHub secrets to include AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY - name: Upload manifest to S3 run: | - aws s3 cp target/manifest.json s3://vhol-datafold-dbt-prod-manifest/manifest.json + aws s3 cp target/manifest.json s3://vhold-prod-manifest-demo-saravula/manifest.json env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_REGION: us-west-2 + AWS_REGION: us-east-1 diff --git a/.github/workflows/dbt_staging.yml b/.github/workflows/dbt_staging.yml index e4ff4bd..c106e02 100644 --- a/.github/workflows/dbt_staging.yml +++ b/.github/workflows/dbt_staging.yml @@ -40,7 +40,7 @@ jobs: # TODO: update your GitHub secrets to include AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY - name: Grab production manifest from S3 run: | - aws s3 cp s3://vhol-datafold-dbt-prod-manifest/manifest.json ./manifest.json + aws s3 cp s3://vhold-prod-manifest-demo-saravula/manifest.json ./manifest.json env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} @@ -60,7 +60,7 @@ jobs: - name: submit artifacts to datafold run: | set -ex - datafold dbt upload --ci-config-id 345 --run-type ${DATAFOLD_RUN_TYPE} --commit-sha ${GIT_SHA} + datafold dbt upload --ci-config-id 400 --run-type ${DATAFOLD_RUN_TYPE} --commit-sha ${GIT_SHA} env: # TODO: update your GitHub secrets to include DATAFOLD_APIKEY DATAFOLD_APIKEY: ${{ secrets.DATAFOLD_APIKEY }} DATAFOLD_RUN_TYPE: "${{ 'pull_request' }}" diff --git a/.gitignore b/.gitignore index b235435..b24fb97 100644 --- a/.gitignore +++ b/.gitignore @@ -162,4 +162,6 @@ cython_debug/ target/ dbt_packages/ -logs/ \ No newline at end of file +logs/ +dbt-env/bin/python +dbt-env/bin/python3 diff --git a/dbt_project.yml b/dbt_project.yml index 595567e..1d2f243 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -27,9 +27,9 @@ clean-targets: # directories to be removed by `dbt clean` # TODO: update the following to your Snowflake account and Datafold Snowflake datasource_id ID vars: data_diff: - prod_database: DEMO - prod_schema: CORE - datasource_id: 4932 + prod_database: DATAFOLD_DEMO + prod_schema: PROD + datasource_id: 7980 # Configuring models # Full documentation: https://docs.getdbt.com/docs/configuring-models diff --git a/models/core/dim_orgs.sql b/models/core/dim_orgs.sql index 73beb44..27eb519 100644 --- a/models/core/dim_orgs.sql +++ b/models/core/dim_orgs.sql @@ -1,3 +1,4 @@ +# WITH orgs AS ( --prod SELECT diff --git a/profiles.yml b/profiles.yml index 1871dfa..68414c1 100644 --- a/profiles.yml +++ b/profiles.yml @@ -7,10 +7,11 @@ vhol_demo: account: "{{ env_var('SNOWFLAKE_ACCOUNT') }}" user: "{{ env_var('SNOWFLAKE_USER') | as_text }}" password: "{{ env_var('SNOWFLAKE_PASSWORD') | as_text }}" - role: DEMO_ROLE - database: DEMO - warehouse: INTEGRATION - schema: "{{ env_var('SNOWFLAKE_SCHEMA') | as_text }}" + role: DATAFOLD_ROLE + database: DATAFOLD_DEMO + warehouse: TRANSFORMING + # schema: "{{ env_var('SNOWFLAKE_SCHEMA') | as_text }}" + schema: TEST_DATAFOLD_TMP threads: 24 prod: @@ -18,8 +19,8 @@ vhol_demo: account: "{{ env_var('SNOWFLAKE_ACCOUNT') }}" user: "{{ env_var('SNOWFLAKE_USER') | as_text }}" password: "{{ env_var('SNOWFLAKE_PASSWORD') | as_text }}" - role: DEMO_ROLE - database: DEMO - warehouse: INTEGRATION - schema: CORE - threads: 24 \ No newline at end of file + role: DATAFOLD_ROLE + database: DATAFOLD_DEMO + warehouse: TRANSFORMING + schema: PROD + threads: 24 diff --git a/requirements.txt b/requirements.txt index 9ffd3fb..ae3194c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -dbt-core==1.7.7 -dbt-snowflake==1.7.1 -data-diff==0.11.0 -datafold-sdk==0.0.19 \ No newline at end of file +dbt-core +dbt-snowflake +data-diff +datafold-sdk