From b779d38ed04de95d7f500585dd07c259092d3373 Mon Sep 17 00:00:00 2001 From: Jayana Gunaweera <79576139+JayanaGunaweera01@users.noreply.github.com> Date: Sat, 20 Apr 2024 14:54:35 +0530 Subject: [PATCH 1/6] feat: bias-mitigation-automation.yml Introduced automation capabilities to the IBM AIF 360. This automation was rigorously tested using various notebooks developed by IBM AIF 360. Developers can now seamlessly integrate automation into their workflow by specifying a desired notebook name for execution, resulting in an output notebook with the .ipynb extension. Signed-off-by: Jayana Gunaweera <79576139+JayanaGunaweera01@users.noreply.github.com> --- .../workflows/bias-mitigation-automation.yml | 217 ++++++++++++++++++ 1 file changed, 217 insertions(+) create mode 100644 .github/workflows/bias-mitigation-automation.yml diff --git a/.github/workflows/bias-mitigation-automation.yml b/.github/workflows/bias-mitigation-automation.yml new file mode 100644 index 00000000..bb0c7383 --- /dev/null +++ b/.github/workflows/bias-mitigation-automation.yml @@ -0,0 +1,217 @@ +name: βš– Bias Mitigation Automation + +# Controls when the action will run. +on: + workflow_dispatch: + inputs: + name: + description: 'πŸ“— Enter a name for your notebook (ex: output_notebook.ipynb )' + required: true + notebook: + description: "πŸ“’ Select the notebook you need to run." + default: "tutorial_medical_expenditure.ipynb" + type: choice + options: + - "tutorial_medical_expenditure.ipynb" + - "tutorial_bias_advertising.ipynb" + - "tutorial_medical_expenditure.ipynb" + - "demo_reweighing_preproc.ipynb" + - "demo_short_gerryfair_test.ipynb" + - "demo_reject_option_classification.ipynb" + - "demo_new_features.ipynb" + - "demo_ot_metric.ipynb" + - "demo_optim_preproc_adult.ipynb" + - "demo_optim_data_preproc.ipynb" + - "demo_meta_classifier.ipynb" + - "demo_mdss_detector.ipynb" + - "demo_mdss_classifier_metric.ipynb" + - "demo_lime.ipynb" + - "demo_lfr.ipynb" + - "demo_json_explainers.ipynb" + - "demo_gerryfair.ipynb" + - "demo_exponentiated_gradient_reduction.ipynb" + - "demo_disparate_impact_remover.ipynb" + - "demo_deterministic_reranking.ipynb" + - "demo_calibrated_eqodds_postprocessing.ipynb" + - "demo_adversarial_debiasing.ipynb" + + # A workflow run is made up of one or more jobs that can run sequentially or in parallel +jobs: + build-py: + runs-on: ubuntu-latest + + strategy: + fail-fast: false + matrix: + python-version: [ 3.11] + + env: + UCI_DB: "https://archive.ics.uci.edu/ml/machine-learning-databases" + PROPUBLICA_GH: "https://raw.githubusercontent.com/propublica/compas-analysis/bafff5da3f2e45eca6c2d5055faad269defd135a" + REPO_KEY: ${{secrets.ETHAI_AUDIT_HUB_GITHUB_TOKEN}} + username: github-actions + + # Steps represent a sequence of tasks that will be executed as part of the job + steps: + - name: πŸ“₯ Check out repo + uses: actions/checkout@v3 + + - name: πŸ“¦ Set up R + uses: r-lib/actions/setup-r@v2 + + # Cache R packages + - name: πŸ’° Cache R packages + uses: actions/cache@v2 + with: + path: ~/.local/lib/R/site-library + key: ${{ runner.os }}-R-${{ hashFiles('**/DESCRIPTION') }} + restore-keys: | + ${{ runner.os }}-R- + + - name: 🐍 Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python-version }} + + # Cache Python packages + - name: πŸ’΅ Cache Python packages + uses: actions/cache@v2 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} + restore-keys: | + ${{ runner.os }}-pip- + + - name: πŸ“’ Echo Inputs + run: | + echo -e "\e[1;34mNotebook Name:\e[0m ${{ github.event.inputs.name }}" + echo -e "\e[1;34mExecuting Notebook:\e[0m ${{ github.event.inputs.notebook }}" + + - name: πŸ“¦ Install dependencies + run: | + python -m pip install --upgrade pip setuptools wheel + pip install -e '.[all]' + pip install flake8 + pip list + python -m rpy2.situation + + - name: πŸ“Š Download basic datasets + run: | + wget ${UCI_DB}/adult/adult.data -P aif360/data/raw/adult/ + wget ${UCI_DB}/adult/adult.test -P aif360/data/raw/adult/ + wget ${UCI_DB}/adult/adult.names -P aif360/data/raw/adult/ + wget ${UCI_DB}/statlog/german/german.data -P aif360/data/raw/german/ + wget ${UCI_DB}/statlog/german/german.doc -P aif360/data/raw/german/ + wget ${PROPUBLICA_GH}/compas-scores-two-years.csv -P aif360/data/raw/compas/ + wget ${UCI_DB}/00222/bank-additional.zip -P aif360/data/raw/bank/ && unzip -j aif360/data/raw/bank/bank-additional.zip -d aif360/data/raw/bank/ && rm aif360/data/raw/bank/bank-additional.zip + (cd aif360/data/raw/meps;Rscript generate_data.R <<< y) + + - name: 🧹Lint with flake8 + run: | + # stop the build if there are Python syntax errors or undefined names + flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics + # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide + flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + + - name: πŸ“’ Divert to Jupyter notebook directory + run: | + cd ${{ github.workspace }}/examples + ls -a + + - name: πŸš€ Execute Jupyter Notebook + run: | + # Navigate to examples directory + cd ${{ github.workspace }}/examples + + # Get path + pwd + + # Create an empty file for capturing the stack trace + touch ipynb_execution.txt + + # Give write access to .txt + chmod +x ipynb_execution.txt + + # Execute the Jupyter notebook and save the output to a new notebook + jupyter nbconvert --to notebook --execute ${{ github.workspace }}/examples/${{ github.event.inputs.notebook }} --output ${{ github.workspace }}/automation/${{ github.event.inputs.name }} 2>&1 | tee -a ipynb_execution.txt || { echo "error: failed to execute Jupyter notebook. Please see log files to see the stacktrace."; exit 1; } + + - name: πŸ“„ Log ipynb execution stack trace + uses: actions/upload-artifact@v2 + with: + name: ipynb-execution-stack-trace + path: ipynb_execution.txt + + - name: πŸ“ commit updated notebook + uses: EndBug/add-and-commit@v7 + with: + author_name: Plot update bot + message: "add: executed notebook" + add: "${{ github.workspace }}/automation/${{ github.event.inputs.name }}" + + # Add a new step to commit and push the changes + - name: πŸ”€ Commit and Push Changes + run: | + git config --local user.name actions-user + git config --local user.email "actions@github.com" + + - name: πŸ“‚ Persist Logs of notebook execution log file + run: | + # Create artifacts directory + mkdir -p ${{ github.workspace }}/automation/artifacts + + # Copy file to artifacts directory + cp ${{ github.workspace }}/examples/ipynb_execution.txt ${{ github.workspace }}/automation/artifacts/ + if: ${{ always() }} + + - name: πŸ“‚ Persist Logs of executed ipynb file + run: | + # Copy file to artifacts directory + cp ${{ github.workspace }}/automation/${{ github.event.inputs.name }} ${{ github.workspace }}/automation/artifacts/ + if: ${{ always() }} + + - name: πŸ“€ Upload Artifacts + uses: actions/upload-artifact@v2 + if: always() + with: + name: "Audit Report ${{ github.actor }} - ${{ github.run_number }} " + path: ${{ github.workspace }}/automation/artifacts + + build-r: + runs-on: ubuntu-latest + + # Define strategy for job execution + strategy: + fail-fast: false + # Matrix strategy allows running multiple configurations + matrix: + python-version: [3.11] + + steps: + # Checkout the repository + - name: πŸ“₯ Check out repo + uses: actions/checkout@v3 + + # Set up R environment + - name: πŸ“¦ Set up R + uses: r-lib/actions/setup-r@v2 + + # Set up Python environment + - name: 🐍 Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python-version }} + + # Install R dependencies + - name: πŸ“¦ Install R dependencies + run: install.packages(c("reticulate", "rstudioapi", "testthat")) + shell: Rscript {0} + + # Install Python dependencies + - name: πŸ“¦ Install Python dependencies + run: | + python -m pip install --upgrade pip setuptools wheel + pip install '.[all]' + + # Install R package + - name: πŸ“¦ Install R package + run: R CMD INSTALL aif360/aif360-r From 08aaae220d2a4a5c4a1117a57e1f38be5b4d33a4 Mon Sep 17 00:00:00 2001 From: Jayana Gunaweera <79576139+JayanaGunaweera01@users.noreply.github.com> Date: Sat, 20 Apr 2024 15:20:13 +0530 Subject: [PATCH 2/6] add: caching steps Added caching steps in setting up R and python packages. Signed-off-by: Jayana Gunaweera <79576139+JayanaGunaweera01@users.noreply.github.com> --- .github/workflows/ci.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 976479af..64828b39 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -86,11 +86,27 @@ jobs: - name: Set up R uses: r-lib/actions/setup-r@v2 + - name: Cache R packages + uses: actions/cache@v2 + with: + path: ~/.local/lib/R/site-library + key: ${{ runner.os }}-R-${{ hashFiles('**/DESCRIPTION') }} + restore-keys: | + ${{ runner.os }}-R- + - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v3 with: python-version: ${{ matrix.python-version }} + - name: Cache Python packages + uses: actions/cache@v2 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} + restore-keys: | + ${{ runner.os }}-pip- + - name: Install R dependencies run: install.packages(c("reticulate", "rstudioapi", "testthat")) shell: Rscript {0} From 864149686e0aa5ec0fd8670ec25d1c57c2954401 Mon Sep 17 00:00:00 2001 From: Jayana Gunaweera <79576139+JayanaGunaweera01@users.noreply.github.com> Date: Sat, 20 Apr 2024 15:31:42 +0530 Subject: [PATCH 3/6] update: ci.yml Signed-off-by: Jayana Gunaweera <79576139+JayanaGunaweera01@users.noreply.github.com> --- .github/workflows/ci.yml | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 64828b39..976479af 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -86,27 +86,11 @@ jobs: - name: Set up R uses: r-lib/actions/setup-r@v2 - - name: Cache R packages - uses: actions/cache@v2 - with: - path: ~/.local/lib/R/site-library - key: ${{ runner.os }}-R-${{ hashFiles('**/DESCRIPTION') }} - restore-keys: | - ${{ runner.os }}-R- - - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v3 with: python-version: ${{ matrix.python-version }} - - name: Cache Python packages - uses: actions/cache@v2 - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} - restore-keys: | - ${{ runner.os }}-pip- - - name: Install R dependencies run: install.packages(c("reticulate", "rstudioapi", "testthat")) shell: Rscript {0} From 4fc49360c63720d28ed8b547066b3711a4303911 Mon Sep 17 00:00:00 2001 From: Jayana Gunaweera <79576139+JayanaGunaweera01@users.noreply.github.com> Date: Sat, 20 Apr 2024 15:36:47 +0530 Subject: [PATCH 4/6] update: ci.yml Added caching steps for python and R setup.This will optimize the workflow by reducing the time to complete a workflow execution. Signed-off-by: Jayana Gunaweera <79576139+JayanaGunaweera01@users.noreply.github.com> --- .github/workflows/ci.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 976479af..2da50a8f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -86,11 +86,27 @@ jobs: - name: Set up R uses: r-lib/actions/setup-r@v2 + - name: Cache R packages + uses: actions/cache@v2 + with: + path: ~/.local/lib/R/site-library + key: ${{ runner.os }}-R-${{ hashFiles('**/DESCRIPTION') }} + restore-keys: | + ${{ runner.os }}-R- + - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v3 with: python-version: ${{ matrix.python-version }} + - name: Cache Python packages + uses: actions/cache@v2 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} + restore-keys: | + ${{ runner.os }}-pip- + - name: Install R dependencies run: install.packages(c("reticulate", "rstudioapi", "testthat")) shell: Rscript {0} @@ -102,3 +118,4 @@ jobs: - name: Install R package run: R CMD INSTALL aif360/aif360-r + From 2e9b4e951d86049175890f38f87b54f598878c70 Mon Sep 17 00:00:00 2001 From: Jayana Gunaweera <79576139+JayanaGunaweera01@users.noreply.github.com> Date: Sat, 20 Apr 2024 15:54:14 +0530 Subject: [PATCH 5/6] update: ci.yml Signed-off-by: Jayana Gunaweera <79576139+JayanaGunaweera01@users.noreply.github.com> --- .github/workflows/ci.yml | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2da50a8f..55612951 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -86,27 +86,11 @@ jobs: - name: Set up R uses: r-lib/actions/setup-r@v2 - - name: Cache R packages - uses: actions/cache@v2 - with: - path: ~/.local/lib/R/site-library - key: ${{ runner.os }}-R-${{ hashFiles('**/DESCRIPTION') }} - restore-keys: | - ${{ runner.os }}-R- - - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v3 with: python-version: ${{ matrix.python-version }} - - name: Cache Python packages - uses: actions/cache@v2 - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} - restore-keys: | - ${{ runner.os }}-pip- - - name: Install R dependencies run: install.packages(c("reticulate", "rstudioapi", "testthat")) shell: Rscript {0} From e0e8bb71fd8c89ef97b1f9db3dc2191763efad51 Mon Sep 17 00:00:00 2001 From: Jayana Gunaweera <79576139+JayanaGunaweera01@users.noreply.github.com> Date: Sat, 20 Apr 2024 15:55:03 +0530 Subject: [PATCH 6/6] update ci.yml added caching to ci-cd pipeline Signed-off-by: Jayana Gunaweera <79576139+JayanaGunaweera01@users.noreply.github.com> --- .github/workflows/ci.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 55612951..2da50a8f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -86,11 +86,27 @@ jobs: - name: Set up R uses: r-lib/actions/setup-r@v2 + - name: Cache R packages + uses: actions/cache@v2 + with: + path: ~/.local/lib/R/site-library + key: ${{ runner.os }}-R-${{ hashFiles('**/DESCRIPTION') }} + restore-keys: | + ${{ runner.os }}-R- + - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v3 with: python-version: ${{ matrix.python-version }} + - name: Cache Python packages + uses: actions/cache@v2 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} + restore-keys: | + ${{ runner.os }}-pip- + - name: Install R dependencies run: install.packages(c("reticulate", "rstudioapi", "testthat")) shell: Rscript {0}