diff --git a/.github/workflows/api-tests.yml b/.github/workflows/api-tests.yml new file mode 100644 index 00000000..cc201983 --- /dev/null +++ b/.github/workflows/api-tests.yml @@ -0,0 +1,55 @@ +name: API tests + + +on: + push: + branches: + - main + pull_request: + branches: + - main + +jobs: + api-tests: + runs-on: ubuntu-latest + environment: testing + env: + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + LANGFUSE_SECRET_KEY: ${{ secrets.LANGFUSE_SECRET_KEY }} + LANGFUSE_PUBLIC_KEY: ${{ secrets.LANGFUSE_PUBLIC_KEY }} + LANGFUSE_HOST: ${{ secrets.LANGFUSE_HOST }} + RXN_CLASSIFICATION_MODEL_PATH: ${{ secrets.RXN_CLASSIFICATION_MODEL_PATH }} + AZ_MODEL_CONFIG_PATH: ${{ secrets.AZ_MODEL_CONFIG_PATH }} + AZ_MODELS_PATH: ${{ secrets.AZ_MODELS_PATH }} + AZURE_AI_API_KEY: ${{ secrets.AZURE_AI_API_KEY }} + AZURE_AI_API_BASE: ${{ secrets.AZURE_AI_API_BASE }} + DEEPINFRA_API_KEY: ${{ secrets.DEEPINFRA_API_KEY }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + TOGETHER_API_KEY: ${{ secrets.TOGETHER_API_KEY }} + FIREWORKS_AI_API_KEY: ${{ secrets.FIREWORKS_AI_API_KEY }} + ENABLE_LOGGING: False + defaults: + run: + working-directory: ./tests/ + strategy: + matrix: + os: [ubuntu-latest] + python-version: [3.9] + steps: + - uses: actions/checkout@v4 + + - name: Set up Python 3.9 + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements_tests.txt + + - name: Run all API tests + run: | + pytest ./api-tests/ -v --ignore=./api-tests/test_api_deepseek.py -v \ No newline at end of file diff --git a/.github/workflows/deepseek-tests.yml b/.github/workflows/deepseek-tests.yml new file mode 100644 index 00000000..1cf976bd --- /dev/null +++ b/.github/workflows/deepseek-tests.yml @@ -0,0 +1,54 @@ +name: Deepseek API tests + + +on: + push: + branches: + - main + workflow_dispatch: + +jobs: + deepseek-api-tests: + runs-on: ubuntu-latest + environment: testing + env: + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + LANGFUSE_SECRET_KEY: ${{ secrets.LANGFUSE_SECRET_KEY }} + LANGFUSE_PUBLIC_KEY: ${{ secrets.LANGFUSE_PUBLIC_KEY }} + LANGFUSE_HOST: ${{ secrets.LANGFUSE_HOST }} + RXN_CLASSIFICATION_MODEL_PATH: ${{ secrets.RXN_CLASSIFICATION_MODEL_PATH }} + AZ_MODEL_CONFIG_PATH: ${{ secrets.AZ_MODEL_CONFIG_PATH }} + AZ_MODELS_PATH: ${{ secrets.AZ_MODELS_PATH }} + AZURE_AI_API_KEY: ${{ secrets.AZURE_AI_API_KEY }} + AZURE_AI_API_BASE: ${{ secrets.AZURE_AI_API_BASE }} + DEEPINFRA_API_KEY: ${{ secrets.DEEPINFRA_API_KEY }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + TOGETHER_API_KEY: ${{ secrets.TOGETHER_API_KEY }} + FIREWORKS_AI_API_KEY: ${{ secrets.FIREWORKS_AI_API_KEY }} + ENABLE_LOGGING: False + defaults: + run: + working-directory: ./tests/ + strategy: + matrix: + os: [ubuntu-latest] + python-version: [3.9] + steps: + - uses: actions/checkout@v4 + + - name: Set up Python 3.9 + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements_tests.txt + + - name: Run deepseek tests + if: github.event_name != 'workflow_dispatch' + run: | + pytest ./api-tests/test_api_deepseek.py -v \ No newline at end of file diff --git a/.github/workflows/llm-tests.yml b/.github/workflows/llm-tests.yml new file mode 100644 index 00000000..f51fd08a --- /dev/null +++ b/.github/workflows/llm-tests.yml @@ -0,0 +1,55 @@ +name: LLM tests + + +on: + push: + branches: + - main + pull_request: + branches: + - main + +jobs: + llm-tests: + runs-on: ubuntu-latest + environment: testing + env: + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + LANGFUSE_SECRET_KEY: ${{ secrets.LANGFUSE_SECRET_KEY }} + LANGFUSE_PUBLIC_KEY: ${{ secrets.LANGFUSE_PUBLIC_KEY }} + LANGFUSE_HOST: ${{ secrets.LANGFUSE_HOST }} + RXN_CLASSIFICATION_MODEL_PATH: ${{ secrets.RXN_CLASSIFICATION_MODEL_PATH }} + AZ_MODEL_CONFIG_PATH: ${{ secrets.AZ_MODEL_CONFIG_PATH }} + AZ_MODELS_PATH: ${{ secrets.AZ_MODELS_PATH }} + AZURE_AI_API_KEY: ${{ secrets.AZURE_AI_API_KEY }} + AZURE_AI_API_BASE: ${{ secrets.AZURE_AI_API_BASE }} + DEEPINFRA_API_KEY: ${{ secrets.DEEPINFRA_API_KEY }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + TOGETHER_API_KEY: ${{ secrets.TOGETHER_API_KEY }} + FIREWORKS_AI_API_KEY: ${{ secrets.FIREWORKS_AI_API_KEY }} + ENABLE_LOGGING: False + defaults: + run: + working-directory: ./tests/ + strategy: + matrix: + os: [ubuntu-latest] + python-version: [3.9] + steps: + - uses: actions/checkout@v4 + + - name: Set up Python 3.9 + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements_tests.txt + + - name: Run LLM tests + run: | + pytest ./llm-tests/ -v \ No newline at end of file diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml deleted file mode 100644 index e1118287..00000000 --- a/.github/workflows/tests.yml +++ /dev/null @@ -1,86 +0,0 @@ -name: unit tests - -on: - push: - branches: - - main - pull_request: - branches: - - main - -jobs: - test: - runs-on: ubuntu-latest - environment: testing - env: - ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} - OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - LANGFUSE_SECRET_KEY: ${{ secrets.LANGFUSE_SECRET_KEY }} - LANGFUSE_PUBLIC_KEY: ${{ secrets.LANGFUSE_PUBLIC_KEY }} - LANGFUSE_HOST: ${{ secrets.LANGFUSE_HOST }} - RXN_CLASSIFICATION_MODEL_PATH: ${{ secrets.RXN_CLASSIFICATION_MODEL_PATH }} - AZ_MODEL_CONFIG_PATH: ${{ secrets.AZ_MODEL_CONFIG_PATH }} - AZ_MODELS_PATH: ${{ secrets.AZ_MODELS_PATH }} - AZURE_AI_API_KEY: ${{ secrets.AZURE_AI_API_KEY }} - AZURE_AI_API_BASE: ${{ secrets.AZURE_AI_API_BASE }} - DEEPINFRA_API_KEY: ${{ secrets.DEEPINFRA_API_KEY }} - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - TOGETHER_API_KEY: ${{ secrets.TOGETHER_API_KEY }} - FIREWORKS_AI_API_KEY: ${{ secrets.FIREWORKS_AI_API_KEY }} - ENABLE_LOGGING: False - defaults: - run: - working-directory: ./tests/ - strategy: - matrix: - os: [ubuntu-latest] - python-version: [3.9] - steps: - - uses: actions/checkout@v4 - - - name: Set up Python 3.9 - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install -r requirements_tests.txt - - - name: Run tests - run: | - pytest -v - - viewer-tests: - runs-on: ubuntu-latest - defaults: - run: - working-directory: ./viewer/ - strategy: - matrix: - os: [ubuntu-latest] - node-version: [16.x] - steps: - - uses: actions/checkout@v4 - - - name: Set up Node.js ${{ matrix.node-version }} - uses: actions/setup-node@v3 - with: - node-version: ${{ matrix.node-version }} - - - name: Create npmrc - run: | - echo "//registry.npmjs.org/:_authToken=" > .npmrc - echo "registry=https://registry.npmjs.org/" >> .npmrc - echo "always-auth=false" >> .npmrc - - - name: Install Jest directly - run: | - npm install --no-package-lock jest@29.7.0 jest-environment-jsdom@29.7.0 - - - name: Run tests - env: - NODE_AUTH_TOKEN: "" - run: npx jest diff --git a/tests/api-tests/__init__.py b/tests/api-tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/api-tests/test_api_call_fail.py b/tests/api-tests/test_api_call_fail.py new file mode 100644 index 00000000..67e26509 --- /dev/null +++ b/tests/api-tests/test_api_call_fail.py @@ -0,0 +1,76 @@ +import requests +import json +import pytest + +import rootutils +root_dir = rootutils.setup_root(".", + indicator=".project-root", + pythonpath=True) + +from tests.variables_test import BASE_URL, ENDPOINTS, X_API_KEY, \ + DEEPSEEK_FIREWORKS_MODEL, USPTO_MODEL, CLAUDE_MODEL, PISTACHIO_MODEL + + +def test_retrosynthesis_fail(): + """Tests retrosynthesis endpoint with empty input("") + + Asserts + ------- + status_code : int + 400 for a failed request. + error message : dict + The response should be a dictionary with keys ['error']. + """ + url = f"{BASE_URL}{ENDPOINTS['retrosynthesis']}" + + payload = json.dumps({ + "advanced_prompt": "True", + "stability_flag": "False", + "hallucination_check": "False", + "llm": CLAUDE_MODEL, + "model_version": PISTACHIO_MODEL}) + + headers = { + 'x-api-key': X_API_KEY, + 'Content-Type': 'application/json' + } + + response = requests.request("POST", url, headers=headers, data=payload) + + assert response.status_code == 400 + assert response.json() == { + "error": "SMILES string is required. Please include a 'smiles' field"} + + +def test_rerun_retro_fail(): + """Tests rerun_retrosynthesis endpoint with empty input("") + + Asserts + ------- + status_code: 400 + error message: status code and error message + """ + url = f"{BASE_URL}{ENDPOINTS['rerun_retro']}" + + payload = json.dumps({ + "advanced_prompt": "True", + "stability_flag": "False", + "hallucination_check": "False", + "llm": DEEPSEEK_FIREWORKS_MODEL, + "model_version": USPTO_MODEL}) + + headers = { + 'x-api-key': X_API_KEY, + 'Content-Type': 'application/json' + } + + response = requests.request("POST", url, headers=headers, data=payload) + + assert response.status_code == 400 + assert response.json() == { + "error": + "Molecule string is required, Please include a 'smiles' field"} + + +if __name__ == '__main__': + pytest.main() diff --git a/tests/api-tests/test_api_deepseek.py b/tests/api-tests/test_api_deepseek.py new file mode 100644 index 00000000..4e6ed05c --- /dev/null +++ b/tests/api-tests/test_api_deepseek.py @@ -0,0 +1,303 @@ +import pytest +import json +import requests + +import rootutils +root_dir = rootutils.setup_root(".", + indicator=".project-root", + pythonpath=True) + +from tests.variables_test import MOLECULE_1, BASE_URL, ENDPOINTS, X_API_KEY, \ + DEEPSEEK_FIREWORKS_MODEL, PISTACHIO_MODEL, USPTO_MODEL + + +def test_retrosynthesis_deepseek_pistachio_p1_success(): + """Test retrosynthesis endpoint with + Model: DeepSeek + Model Version: Pistachio + Prompt: Advance. + + Asserts + ------- + status_code : int + 200 for a successful request. + response : dict + The response should be a dictionary with keys + ['dependencies', 'steps']. + """ + url = f"{BASE_URL}{ENDPOINTS['retrosynthesis']}" + + payload = json.dumps({ + "smiles": MOLECULE_1, + "stability_flag": "False", + "hallucination_check": "False", + "advanced_prompt": "True", + "llm": DEEPSEEK_FIREWORKS_MODEL, + "model_version": PISTACHIO_MODEL}) + + headers = { + 'x-api-key': X_API_KEY, + 'Content-Type': 'application/json' + } + + response = requests.request("POST", url, headers=headers, data=payload) + + assert response.status_code == 200 + assert isinstance(response.json(), dict) + assert ['dependencies', 'steps'] == list(response.json().keys()) + + +def test_retrosynthesis_deepseek_pistachio_p0_success(): + """Test retrosynthesis endpoint with + Model: DeepSeek + Model Version: Pistachio + Prompt: Basic. + + Asserts + ------- + status_code : int + 200 for a successful request. + response : dict + The response should be a dictionary with keys + ['dependencies', 'steps']. + """ + url = f"{BASE_URL}{ENDPOINTS['retrosynthesis']}" + + payload = json.dumps({ + "smiles": MOLECULE_1, + "stability_flag": "False", + "hallucination_check": "False", + "advanced_prompt": "False", + "llm": DEEPSEEK_FIREWORKS_MODEL, + "model_version": PISTACHIO_MODEL}) + + headers = { + 'x-api-key': X_API_KEY, + 'Content-Type': 'application/json' + } + + response = requests.request("POST", url, headers=headers, data=payload) + + assert response.status_code == 200 + assert isinstance(response.json(), dict) + assert ['dependencies', 'steps'] == list(response.json().keys()) + + +def test_rerun_retro_deepseek_pistachio_p1_success(): + """Test rerun_retro endpoint with + Model: DeepSeek + Model Version: Pistachio + Prompt: Advance. + + Asserts + ------- + status_code : int + 200 for a successful request. + response : dict + The response should be a dictionary with keys + ['dependencies', 'steps']. + """ + url = f"{BASE_URL}{ENDPOINTS['rerun_retro']}" + + payload = json.dumps({ + "smiles": MOLECULE_1, + "stability_flag": "False", + "hallucination_check": "False", + "advanced_prompt": "True", + "llm": DEEPSEEK_FIREWORKS_MODEL, + "model_version": PISTACHIO_MODEL}) + + headers = { + 'x-api-key': X_API_KEY, + 'Content-Type': 'application/json' + } + + response = requests.request("POST", url, headers=headers, data=payload) + + assert response.status_code == 200 + assert isinstance(response.json(), dict) + assert ['dependencies', 'steps'] == list(response.json().keys()) + + +def test_rerun_retro_deepseek_pistachio_p0_success(): + """Test rerun_retro endpoint with + Model: DeepSeek + Model Version: Pistachio + Prompt: Basic. + + Asserts + ------- + status_code : int + 200 for a successful request. + response : dict + The response should be a dictionary with keys + ['dependencies', 'steps']. + """ + url = f"{BASE_URL}{ENDPOINTS['rerun_retro']}" + + payload = json.dumps({ + "smiles": MOLECULE_1, + "stability_flag": "False", + "hallucination_check": "False", + "advanced_prompt": "False", + "llm": DEEPSEEK_FIREWORKS_MODEL, + "model_version": PISTACHIO_MODEL}) + + headers = { + 'x-api-key': X_API_KEY, + 'Content-Type': 'application/json' + } + + response = requests.request("POST", url, headers=headers, data=payload) + + assert response.status_code == 200 + assert isinstance(response.json(), dict) + assert ['dependencies', 'steps'] == list(response.json().keys()) + + +def test_rerun_retro_deepseek_uspto_p1_success(): + """Test rerun_retro endpoint with + Model: DeepSeek + Model Version: USPTO + Prompt: Advance. + + Asserts + ------- + status_code : int + 200 for a successful request. + response : dict + The response should be a dictionary with keys + ['dependencies', 'steps']. + """ + url = f"{BASE_URL}{ENDPOINTS['rerun_retro']}" + + payload = json.dumps({ + "smiles": MOLECULE_1, + "stability_flag": "False", + "hallucination_check": "False", + "advanced_prompt": "True", + "llm": DEEPSEEK_FIREWORKS_MODEL, + "model_version": USPTO_MODEL}) + + headers = { + 'x-api-key': X_API_KEY, + 'Content-Type': 'application/json' + } + + response = requests.request("POST", url, headers=headers, data=payload) + + assert response.status_code == 200 + assert isinstance(response.json(), dict) + assert ['dependencies', 'steps'] == list(response.json().keys()) + + +def test_rerun_retro_deepseek_uspto_p0_success(): + """Test rerun_retro endpoint with + Model: DeepSeek + Model Version: USPTO + Prompt: Basic. + + Asserts + ------- + status_code : int + 200 for a successful request. + response : dict + The response should be a dictionary with keys + ['dependencies', 'steps']. + """ + url = f"{BASE_URL}{ENDPOINTS['rerun_retro']}" + + payload = json.dumps({ + "smiles": MOLECULE_1, + "stability_flag": "False", + "hallucination_check": "False", + "advanced_prompt": "False", + "llm": DEEPSEEK_FIREWORKS_MODEL, + "model_version": USPTO_MODEL}) + + headers = { + 'x-api-key': X_API_KEY, + 'Content-Type': 'application/json' + } + + response = requests.request("POST", url, headers=headers, data=payload) + + assert response.status_code == 200 + assert isinstance(response.json(), dict) + assert ['dependencies', 'steps'] == list(response.json().keys()) + + +def test_retrosynthesis_deepseek_uspto_p1_success(): + """Test retrosynthesis endpoint with + Model: DeepSeek + Model Version: USPTO + Prompt: Advance. + + Asserts + ------- + status_code : int + 200 for a successful request. + response : dict + The response should be a dictionary with keys + ['dependencies', 'steps']. + """ + url = f"{BASE_URL}{ENDPOINTS['retrosynthesis']}" + + payload = json.dumps({ + "smiles": MOLECULE_1, + "stability_flag": "False", + "hallucination_check": "False", + "advanced_prompt": "True", + "llm": DEEPSEEK_FIREWORKS_MODEL, + "model_version": USPTO_MODEL}) + + headers = { + 'x-api-key': X_API_KEY, + 'Content-Type': 'application/json' + } + + response = requests.request("POST", url, headers=headers, data=payload) + + assert response.status_code == 200 + assert isinstance(response.json(), dict) + assert ['dependencies', 'steps'] == list(response.json().keys()) + + +def test_retrosynthesis_deepseek_uspto_p0_success(): + """Test retrosynthesis endpoint with + Model: DeepSeek + Model Version: USPTO + Prompt: Basic. + + Asserts + ------- + status_code : int + 200 for a successful request. + response : dict + The response should be a dictionary with keys + ['dependencies', 'steps']. + """ + url = f"{BASE_URL}{ENDPOINTS['retrosynthesis']}" + + payload = json.dumps({ + "smiles": MOLECULE_1, + "stability_flag": "False", + "hallucination_check": "False", + "advanced_prompt": "False", + "llm": DEEPSEEK_FIREWORKS_MODEL, + "model_version": USPTO_MODEL}) + + headers = { + 'x-api-key': X_API_KEY, + 'Content-Type': 'application/json' + } + + response = requests.request("POST", url, headers=headers, data=payload) + + assert response.status_code == 200 + assert isinstance(response.json(), dict) + assert ['dependencies', 'steps'] == list(response.json().keys()) + + +if __name__ == '__main__': + pytest.main() diff --git a/tests/api-tests/test_api_health.py b/tests/api-tests/test_api_health.py new file mode 100644 index 00000000..015f0f7e --- /dev/null +++ b/tests/api-tests/test_api_health.py @@ -0,0 +1,46 @@ +import requests +import json +import pytest + +import rootutils +root_dir = rootutils.setup_root(".", + indicator=".project-root", + pythonpath=True) + +from tests.variables_test import MOLECULE_1, BASE_URL, ENDPOINTS, X_API_KEY, \ + DEEPSEEK_FIREWORKS_MODEL + + +def test_health_deepseek_success(): + """Test the health of the endpoint. + Asserts + ------- + status_code : int + 200 for a successful request. + response : dict + The response should be a dictionary with {'status' : 'healthy'}. + """ + + url = f"{BASE_URL}{ENDPOINTS['health']}" + + payload = json.dumps({ + "smiles": MOLECULE_1, + "stability_flag": "False", + "hallucination_check": "False", + "advanced_prompt": "False", + "llm": DEEPSEEK_FIREWORKS_MODEL, + "model_version": "USPTO"}) + + headers = { + 'x-api-key': X_API_KEY, + 'Content-Type': 'application/json' + } + + response = requests.request("GET", url, headers=headers, data=payload) + + assert response.status_code == 200 + assert response.json() == {'status': 'healthy'} + + +if __name__ == '__main__': + pytest.main() diff --git a/tests/api-tests/test_api_pistachio_rerun_retro.py b/tests/api-tests/test_api_pistachio_rerun_retro.py new file mode 100644 index 00000000..6ba0d1fb --- /dev/null +++ b/tests/api-tests/test_api_pistachio_rerun_retro.py @@ -0,0 +1,61 @@ +import requests +import json +import pytest + +import rootutils +root_dir = rootutils.setup_root(".", + indicator=".project-root", + pythonpath=True) + +from tests.variables_test import MOLECULE_1, BASE_URL, ENDPOINTS, X_API_KEY, \ + PISTACHIO_MODEL, CLAUDE_MODEL + + +def test_rerun_retro_claude_pistachio_p1_success(): + url = f"{BASE_URL}{ENDPOINTS['rerun_retro']}" + + payload = json.dumps({ + "smiles": MOLECULE_1, + "stability_flag": "False", + "hallucination_check": "False", + "advanced_prompt": "True", + "llm": CLAUDE_MODEL, + "model_version": PISTACHIO_MODEL}) + + headers = { + 'x-api-key': X_API_KEY, + 'Content-Type': 'application/json' + } + + response = requests.request("POST", url, headers=headers, data=payload) + + assert response.status_code == 200 + assert isinstance(response.json(), dict) + assert ['dependencies', 'steps'] == list(response.json().keys()) + + +def test_rerun_retro_claude_pistachio_p0_success(): + url = f"{BASE_URL}{ENDPOINTS['rerun_retro']}" + + payload = json.dumps({ + "smiles": MOLECULE_1, + "stability_flag": "False", + "hallucination_check": "False", + "advanced_prompt": "False", + "llm": CLAUDE_MODEL, + "model_version": PISTACHIO_MODEL}) + + headers = { + 'x-api-key': X_API_KEY, + 'Content-Type': 'application/json' + } + + response = requests.request("POST", url, headers=headers, data=payload) + + assert response.status_code == 200 + assert isinstance(response.json(), dict) + assert ['dependencies', 'steps'] == list(response.json().keys()) + + +if __name__ == '__main__': + pytest.main() diff --git a/tests/api-tests/test_api_pistachio_retrosynthesis.py b/tests/api-tests/test_api_pistachio_retrosynthesis.py new file mode 100644 index 00000000..b8a03676 --- /dev/null +++ b/tests/api-tests/test_api_pistachio_retrosynthesis.py @@ -0,0 +1,61 @@ +import requests +import json +import pytest + +import rootutils +root_dir = rootutils.setup_root(".", + indicator=".project-root", + pythonpath=True) + +from tests.variables_test import MOLECULE_1, BASE_URL, ENDPOINTS, X_API_KEY, \ + PISTACHIO_MODEL, CLAUDE_MODEL + + +def test_retrosynthesis_pistachio_claude_m1p1_success(): + url = f"{BASE_URL}{ENDPOINTS['retrosynthesis']}" + + payload = json.dumps({ + "smiles": MOLECULE_1, + "stability_flag": "False", + "hallucination_check": "False", + "advanced_prompt": "True", + "llm": CLAUDE_MODEL, + "model_version": PISTACHIO_MODEL}) + + headers = { + 'x-api-key': X_API_KEY, + 'Content-Type': 'application/json' + } + + response = requests.request("POST", url, headers=headers, data=payload) + + assert response.status_code == 200 + assert isinstance(response.json(), dict) + assert ['dependencies', 'steps'] == list(response.json().keys()) + + +def test_retrosynthesis_pistachio_claude_m1p0_success(): + url = f"{BASE_URL}{ENDPOINTS['retrosynthesis']}" + + payload = json.dumps({ + "smiles": MOLECULE_1, + "stability_flag": "False", + "hallucination_check": "False", + "advanced_prompt": "False", + "llm": CLAUDE_MODEL, + "model_version": PISTACHIO_MODEL}) + + headers = { + 'x-api-key': X_API_KEY, + 'Content-Type': 'application/json' + } + + response = requests.request("POST", url, headers=headers, data=payload) + + assert response.status_code == 200 + assert isinstance(response.json(), dict) + assert ['dependencies', 'steps'] == list(response.json().keys()) + + +if __name__ == '__main__': + pytest.main() diff --git a/tests/api-tests/test_api_uspto_rerun_retro.py b/tests/api-tests/test_api_uspto_rerun_retro.py new file mode 100644 index 00000000..6ef16811 --- /dev/null +++ b/tests/api-tests/test_api_uspto_rerun_retro.py @@ -0,0 +1,61 @@ +import requests +import json +import pytest + +import rootutils +root_dir = rootutils.setup_root(".", + indicator=".project-root", + pythonpath=True) + +from tests.variables_test import MOLECULE_1, BASE_URL, ENDPOINTS, X_API_KEY, \ + USPTO_MODEL, CLAUDE_MODEL + + +def test_rerun_retro_claude_uspto_p1_success(): + url = f"{BASE_URL}{ENDPOINTS['rerun_retro']}" + + payload = json.dumps({ + "smiles": MOLECULE_1, + "stability_flag": "False", + "hallucination_check": "False", + "advanced_prompt": "True", + "llm": CLAUDE_MODEL, + "model_version": USPTO_MODEL}) + + headers = { + 'x-api-key': X_API_KEY, + 'Content-Type': 'application/json' + } + + response = requests.request("POST", url, headers=headers, data=payload) + + assert response.status_code == 200 + assert isinstance(response.json(), dict) + assert ['dependencies', 'steps'] == list(response.json().keys()) + + +def test_rerun_retro_claude_uspto_p0_success(): + url = f"{BASE_URL}{ENDPOINTS['rerun_retro']}" + + payload = json.dumps({ + "smiles": MOLECULE_1, + "stability_flag": "False", + "hallucination_check": "False", + "advanced_prompt": "False", + "llm": CLAUDE_MODEL, + "model_version": USPTO_MODEL}) + + headers = { + 'x-api-key': X_API_KEY, + 'Content-Type': 'application/json' + } + + response = requests.request("POST", url, headers=headers, data=payload) + + assert response.status_code == 200 + assert isinstance(response.json(), dict) + assert ['dependencies', 'steps'] == list(response.json().keys()) + + +if __name__ == '__main__': + pytest.main() diff --git a/tests/api-tests/test_api_uspto_retrosynthesis.py b/tests/api-tests/test_api_uspto_retrosynthesis.py new file mode 100644 index 00000000..a93d2eff --- /dev/null +++ b/tests/api-tests/test_api_uspto_retrosynthesis.py @@ -0,0 +1,61 @@ +import requests +import json +import pytest + +import rootutils +root_dir = rootutils.setup_root(".", + indicator=".project-root", + pythonpath=True) + +from tests.variables_test import MOLECULE_1, BASE_URL, ENDPOINTS, X_API_KEY, \ + USPTO_MODEL, CLAUDE_MODEL + + +def test_retrosynthesis_uspto_claude_p1_success(): + url = f"{BASE_URL}{ENDPOINTS['retrosynthesis']}" + + payload = json.dumps({ + "smiles": MOLECULE_1, + "stability_flag": "False", + "hallucination_check": "False", + "advanced_prompt": "True", + "llm": CLAUDE_MODEL, + "model_version": USPTO_MODEL}) + + headers = { + 'x-api-key': X_API_KEY, + 'Content-Type': 'application/json' + } + + response = requests.request("POST", url, headers=headers, data=payload) + + assert response.status_code == 200 + assert isinstance(response.json(), dict) + assert ['dependencies', 'steps'] == list(response.json().keys()) + + +def test_retrosynthesis_uspto_claude_p0_success(): + url = f"{BASE_URL}{ENDPOINTS['retrosynthesis']}" + + payload = json.dumps({ + "smiles": MOLECULE_1, + "stability_flag": "False", + "hallucination_check": "False", + "advanced_prompt": "False", + "llm": CLAUDE_MODEL, + "model_version": USPTO_MODEL}) + + headers = { + 'x-api-key': X_API_KEY, + 'Content-Type': 'application/json' + } + + response = requests.request("POST", url, headers=headers, data=payload) + + assert response.status_code == 200 + assert isinstance(response.json(), dict) + assert ['dependencies', 'steps'] == list(response.json().keys()) + + +if __name__ == '__main__': + pytest.main() diff --git a/tests/llm-tests/__init__.py b/tests/llm-tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/test_adv_prompt.py b/tests/llm-tests/test_adv_prompt.py similarity index 54% rename from tests/test_adv_prompt.py rename to tests/llm-tests/test_adv_prompt.py index 7cbe2b34..93d80ff7 100644 --- a/tests/test_adv_prompt.py +++ b/tests/llm-tests/test_adv_prompt.py @@ -1,5 +1,3 @@ -import os -import ast import pytest import rootutils @@ -9,32 +7,25 @@ from src.utils.llm import call_LLM -from tests.variables_test import VALID_SMILE_STRING, DEEPSEEK_FIREWORKS_MODEL, CLAUDE_ADV_MODEL +from tests.variables_test import VALID_SMILE_STRING, \ + DEEPSEEK_FIREWORKS_MODEL, CLAUDE_MODEL def test_claude_adv_success(): - """Tests call_LLM function with advance claude model. + """Tests call_LLM function with claude model. """ status_code, res_text = call_LLM(molecule=VALID_SMILE_STRING, - LLM=CLAUDE_ADV_MODEL) + LLM=CLAUDE_MODEL) if not res_text: print("res_text is empty") status_code = 400 - - assert status_code == 200 - -# OpenAI tests are commented, because OpenAI models are not being used. -# def test_openai_adv_success(): -# status_code, res_text = call_LLM(molecule=VALID_SMILE_STRING, -# LLM=OPENAI_MODEL_ADV) -# if not res_text: -# assert status_code == 404 -# assert status_code == 200 + assert status_code == 200 def test_deepseek_adv_success(): - """Tests call_LLM function with advance deepseek model(hosted in fireworks). + """Tests call_LLM function with + deepseek model(hosted in fireworks). """ status_code, res_text = call_LLM(molecule="CC1=NC=C(N1CCO)[N+]([O-])=O", @@ -46,6 +37,5 @@ def test_deepseek_adv_success(): assert status_code == 200 - if __name__ == '__main__': pytest.main() diff --git a/tests/test_llm.py b/tests/llm-tests/test_llm.py similarity index 54% rename from tests/test_llm.py rename to tests/llm-tests/test_llm.py index 1a73c880..ef886e7f 100644 --- a/tests/test_llm.py +++ b/tests/llm-tests/test_llm.py @@ -1,36 +1,37 @@ -import os -import ast +from src.utils.llm import call_LLM, split_cot_json, split_json_deepseek +from dotenv import load_dotenv import pytest import rootutils -root_dir = rootutils.setup_root(".", indicator=".project-root", pythonpath=True) +root_dir = rootutils.setup_root(".", + indicator=".project-root", + pythonpath=True) -from dotenv import load_dotenv load_dotenv() -from src.utils.llm import call_LLM, split_cot_json, split_json_deepseek SMALL_SMILE_STRING = "CC(=O)O" LARGE_SMILE_STRING = "CC(N)C(=O)NC1=C(C)C=CC=C1C" + def test_call_llm_success(): """Tests call_LLM function with valid smile string. - + Expected output: status_code: 200. res_text: str. """ - from tests.variables_test import VALID_SMILE_STRING - status_code, res_text = call_LLM(molecule=LARGE_SMILE_STRING) - + assert status_code == 200 assert isinstance(res_text, str) def test_split_cot_json_success(): - """Tests split_cot_json function with valid response. split_cot_json() splits the response based on , and tag. - + """Tests split_cot_json function with valid response. + split_cot_json() splits the response based on , + and tag. + Expected output: status_code: 200. thinking_steps: list containing items @@ -38,8 +39,9 @@ def test_split_cot_json_success(): """ from tests.variables_test import VALID_CLAUDE_RESPONSE - status_code, thinking_steps, json_content = split_cot_json(VALID_CLAUDE_RESPONSE) - + status_code, thinking_steps, json_content = split_cot_json( + VALID_CLAUDE_RESPONSE) + assert status_code == 200 assert isinstance(thinking_steps, list) assert thinking_steps @@ -49,16 +51,17 @@ def test_split_cot_json_success(): def test_split_cot_json_fail_501(): """Tests split_cot_json function with empty response. - + Expected output: status_code: 501 thinking_steps: [] json_content: "" """ from tests.variables_test import EMPTY_RESPONSE - - status_code, thinking_steps, json_content = split_cot_json(EMPTY_RESPONSE) - + + status_code, thinking_steps, json_content = split_cot_json( + EMPTY_RESPONSE) + assert status_code == 501 assert thinking_steps == [] assert json_content == "" @@ -66,32 +69,35 @@ def test_split_cot_json_fail_501(): def test_call_llm_deepseek_success(): '''Testing deepseek model, hosted in fireworks. - + Expected output: status_code: 200. res_text: str. ''' from tests.variables_test import DEEPSEEK_FIREWORKS_MODEL - status_code, res_text = call_LLM(molecule=LARGE_SMILE_STRING, LLM=DEEPSEEK_FIREWORKS_MODEL) - + status_code, res_text = call_LLM( + molecule=LARGE_SMILE_STRING, LLM=DEEPSEEK_FIREWORKS_MODEL) + assert status_code == 200 assert isinstance(res_text, str) assert res_text def test_split_json_deepseek_success(): - """Tests split_json_deepseek function with valid response. split_json_deepseek splits the response based on and tag. - + """Tests split_json_deepseek function with valid response. + split_json_deepseek splits the response based on and tag. + Expected output: status_code: 200. thinking_steps: list containing items json_content: str """ from tests.variables_test import DEEPSEEK_ADV_VALID_RESPONSE - - status_code, thinking_steps, json_content = split_json_deepseek(DEEPSEEK_ADV_VALID_RESPONSE) - + + status_code, thinking_steps, json_content = split_json_deepseek( + DEEPSEEK_ADV_VALID_RESPONSE) + assert status_code == 200 assert isinstance(thinking_steps, list) assert isinstance(json_content, str) @@ -99,59 +105,21 @@ def test_split_json_deepseek_success(): def test_split_json_deepseek_fail_503(): """Tests split_json_deepseek function with empty response. - + Expected output: status_code: 503 thinking_step: [] json_content: "" """ from tests.variables_test import EMPTY_RESPONSE - - status_code, thinking_steps, json_content = split_json_deepseek(EMPTY_RESPONSE) - + + status_code, thinking_steps, json_content = split_json_deepseek( + EMPTY_RESPONSE) + assert status_code == 503 assert thinking_steps == [] assert json_content == "" -# OpenAI tests -# Open AI tests are commented out because OpenAI models are not being used in the prod. -# def test_call_llm_openai_success(): - -# from tests.variables_test import VALID_SMILE_STRING - -# status_code, _ = call_LLM(molecule=VALID_SMILE_STRING, LLM="gpt-4o") -# assert status_code == 200 - -# def test_all_openai_success(): - -# from tests.variables_test import VALID_SMILE_STRING - -# successful_tests = [] -# failed_tests = [] -# print("models: ", OPENAI_MODELS) -# for model in OPENAI_MODELS: -# try: -# status_code, res_text = call_LLM(molecule=VALID_SMILE_STRING, LLM=model) -# assert status_code == 200 -# successful_tests.append(model) -# except Exception as e: -# print(f"Error: {e}\n Model: {model}") -# failed_tests.append(model) -# if failed_tests: -# print(f"Failed models: {failed_tests}") - -# def test_split_json_openai_success(): - -# from tests.variables_test import VALID_SMILE_STRING - -# status_code, _ = call_LLM(molecule=VALID_SMILE_STRING, LLM="gpt-4o") -# assert status_code == 200 - -# def test_split_json_openai_fail_502(): - -# status_code, _ = split_json_openAI("") -# assert status_code == 502 - if __name__ == '__main__': pytest.main() diff --git a/tests/test_llm_parsers.py b/tests/llm-tests/test_llm_parsers.py similarity index 84% rename from tests/test_llm_parsers.py rename to tests/llm-tests/test_llm_parsers.py index 2d549341..f704bcbe 100644 --- a/tests/test_llm_parsers.py +++ b/tests/llm-tests/test_llm_parsers.py @@ -1,21 +1,19 @@ -import os -import ast +from dotenv import load_dotenv import pytest import rootutils -root_dir = rootutils.setup_root(".", indicator=".project-root", pythonpath=True) +root_dir = rootutils.setup_root( + ".", indicator=".project-root", pythonpath=True) -from dotenv import load_dotenv load_dotenv() - SMALL_SMILE_STRING = "CC(=O)O" LARGE_SMILE_STRING = "CC(N)C(=O)NC1=C(C)C=CC=C1C" def test_split_json_master_success(): """Tests split_json_master function with valid response. split_json_master call the split_cot_json, split_json_deepseek and split_json_openAI functions, based on the model passed as argument. - + Expected output: status_code: 200. thinking_steps: list, containing items. @@ -24,8 +22,9 @@ def test_split_json_master_success(): from tests.variables_test import VALID_CLAUDE_RESPONSE, CLAUDE_MODEL from src.utils.llm import split_json_master - status_code, thinking_steps, json_content = split_json_master(VALID_CLAUDE_RESPONSE, model=CLAUDE_MODEL) - + status_code, thinking_steps, json_content = split_json_master( + VALID_CLAUDE_RESPONSE, model=CLAUDE_MODEL) + assert status_code == 200 assert isinstance(thinking_steps, list) assert thinking_steps @@ -41,10 +40,11 @@ def test_split_json_master_fail(): thinking_step: [] json_content: "" """ - from tests.variables_test import EMPTY_RESPONSE, CLAUDE_ADV_MODEL + from tests.variables_test import EMPTY_RESPONSE, CLAUDE_MODEL from src.utils.llm import split_json_master - status_code, thinking_steps, json_content = split_json_master(EMPTY_RESPONSE, model=CLAUDE_ADV_MODEL) + status_code, thinking_steps, json_content = split_json_master( + EMPTY_RESPONSE, model=CLAUDE_MODEL) assert status_code == 501 assert thinking_steps == [] assert json_content == "" @@ -59,14 +59,14 @@ def test_validate_json_success_200(): res_explanations: list containing items. res_confidence: list containing items. """ - from tests.variables_test import VALID_CLAUDE_RESPONSE, CLAUDE_ADV_MODEL + from tests.variables_test import VALID_CLAUDE_RESPONSE, CLAUDE_MODEL from src.utils.llm import validate_split_json, split_json_master status_code, _, json_content = split_json_master( - VALID_CLAUDE_RESPONSE, CLAUDE_ADV_MODEL) + VALID_CLAUDE_RESPONSE, CLAUDE_MODEL) - status_code, res_molecules, res_explanations, res_confidence = validate_split_json( - json_content) + status_code, res_molecules, res_explanations, res_confidence = \ + validate_split_json(json_content) assert status_code == 200 assert isinstance(res_molecules, list) @@ -92,8 +92,8 @@ def test_validate_json_fail(): status_code, _, json_content = split_json_master( EMPTY_RESPONSE, CLAUDE_MODEL) - status_code, res_molecules, res_explanations, res_confidence = validate_split_json( - json_content) + status_code, res_molecules, res_explanations, res_confidence = \ + validate_split_json(json_content) assert status_code == 504 assert res_molecules == [] @@ -102,21 +102,24 @@ def test_validate_json_fail(): def test_validity_check_success(): - """Tests validity_check with valid smile string. validity_check checks the validity of the molecules obtained from LLM. + """Tests validity_check with valid smile string. + validity_check checks the validity of the molecules obtained from LLM. Expected Output: output_pathways: list containing output pathways. output_explanations: list containing explanations to output pathways. output_confidence: list containing confidence score. """ - from tests.variables_test import VALID_CLAUDE_RESPONSE, CLAUDE_MODEL, VALID_SMILE_STRING - from src.utils.llm import validate_split_json, split_json_master, validity_check + from tests.variables_test import VALID_CLAUDE_RESPONSE, CLAUDE_MODEL, \ + VALID_SMILE_STRING + from src.utils.llm import validate_split_json, split_json_master, \ + validity_check _, _, json_content = split_json_master( VALID_CLAUDE_RESPONSE, CLAUDE_MODEL) - _, res_molecules, res_explanations, res_confidence = validate_split_json( - json_content) + _, res_molecules, res_explanations, res_confidence = \ + validate_split_json(json_content) output_pathways, output_explanations, output_confidence = validity_check( VALID_SMILE_STRING, res_molecules, res_explanations, res_confidence) @@ -138,13 +141,14 @@ def test_validity_check_fail(): output_confidence: [] """ from tests.variables_test import CLAUDE_MODEL, EMPTY_RESPONSE - from src.utils.llm import validate_split_json, split_json_master, validity_check + from src.utils.llm import validate_split_json, split_json_master, \ + validity_check _, _, json_content = split_json_master( EMPTY_RESPONSE, CLAUDE_MODEL) - _, res_molecules, res_explanations, res_confidence = validate_split_json( - json_content) + _, res_molecules, res_explanations, res_confidence = \ + validate_split_json(json_content) output_pathways, output_explanations, output_confidence = validity_check( "", res_molecules, res_explanations, res_confidence) diff --git a/tests/variables_test.py b/tests/variables_test.py index 8b20023a..f006832b 100644 --- a/tests/variables_test.py +++ b/tests/variables_test.py @@ -4,20 +4,20 @@ # Claude model CLAUDE_MODEL = "claude-3-opus-20240229" -CLAUDE_ADV_MODEL = "claude-3-opus-20240229:adv" - # OpenAI model OPENAI_MODEL = "gpt-4o" -OPENAI_ADV_MODEL = "gpt-4o:adv" +OPENAI_ADV_MODEL = "gpt-4o" # Deepseek model -DEEPSEEK_MODEL = "deepinfra/deepseek-ai/DeepSeek-R1" +DEEPSEEK_DEEPINFRA_MODEL = "deepinfra/deepseek-ai/DeepSeek-R1" -DEEPSEEK_ADV_MODEL = "deepinfra/deepseek-ai/DeepSeek-R1:adv" +DEEPSEEK_FIREWORKS_MODEL = "fireworks_ai/accounts/fireworks/models/deepseek-r1" -DEEPSEEK_FIREWORKS_MODEL = "fireworks_ai/accounts/fireworks/models/deepseek-r1:adv" +# AZ Model +USPTO_MODEL = "USPTO" +PISTACHIO_MODEL = "Pistachio_25" # Valid claude model response VALID_CLAUDE_RESPONSE = 'Here is the single-step retrosynthesis analysis for the molecule CC(=O)CC:\n\n\n\nThe target molecule CC(=O)CC contains a ketone functional group. Possible retrosynthetic disconnections to consider are:\n1) Disconnection of the C-C bond adjacent to the ketone, which could arise from an aldol condensation reaction.\n2) Disconnection of the C-C bond on the other side of the ketone, which could come from a Grignard addition to a carboxylic acid derivative like an ester.\n3) Reduction of the ketone to an alcohol, which could then be derived from an oxidation of the corresponding secondary alcohol.\n\n\n\nFor the aldol disconnection, the precursors would be acetone (CC(=O)C) and acetaldehyde (CC=O). The reaction would proceed via enolate formation of the acetone, followed by nucleophilic addition to the acetaldehyde. A subsequent dehydration step would give the α,β-unsaturated ketone product.\n\n\n\nFor the Grignard addition, the precursors would be propanoyl chloride (CCC(=O)Cl) and methylmagnesium bromide (CMgBr). The Grignard reagent would add to the carbonyl, followed by an acidic workup to give the final ketone product. \n\n\n\nFor the alcohol reduction, the precursor would be butan-2-ol (CC(O)CC). Oxidation, potentially using a chromium reagent like pyridinium chlorochromate (PCC) or a Swern oxidation, would convert the secondary alcohol to the ketone.\n\n\n\n\n\n{\n "data": [\n ["CC(=O)C", "CC=O"],\n ["CCC(=O)Cl", "CMgBr"],\n ["CC(O)CC"]\n ],\n "explanation": [\n "Aldol condensation of acetone and acetaldehyde, proceeding via enolate formation, nucleophilic addition, and dehydration",\n "Grignard addition of methylmagnesium bromide to propanoyl chloride, followed by acidic workup",\n "Oxidation of butan-2-ol, e.g. using PCC or Swern conditions"\n ],\n "confidence_scores": [\n 0.9,\n 0.7,\n 0.8\n ]\n}\n' @@ -676,3 +676,16 @@ "Sulfadiazine": "NC1=CC=C(C=C1)S(=O)(=O)NC1=NC=CC=N1", "Phenprocoumon": "CCC(C1=CC=CC=C1)C1=C(O)C2=C(OC1=O)C=CC=C2", } + + +BASE_URL = "http://ec2-18-220-15-234.us-east-2.compute.amazonaws.com:5000" + +ENDPOINTS = { + "retrosynthesis": "/api/retrosynthesis", + "rerun_retro": "/api/rerun_retrosynthesis", + "health": "/api/health" +} + +MOLECULE_1 = "COC1=CC(C(O)C(C)N)=C(OC)C=C1" + +X_API_KEY = "your-secure-api-key"