Migrate the unified compiler to Catalyst

.codecov.yml

@@ -7,3 +7,7 @@ coverage:
   status:
     project: false
     patch: true
+
+# TODO: add coverage for the python compiler interface
+ignore:
+  - "frontend/catalyst/python_interface"

.github/workflows/build-wheel-linux-arm64.yaml

@@ -501,3 +501,16 @@ jobs:
       if: env.BUILD_STANDALONE_PLUGIN == 'true'
       run: |
         python${{ matrix.python_version }} -m pytest standalone_plugin_wheel/standalone_plugin/test -n auto
+
+    - name: Install Unified Compiler Dependencies
+      run: |
+        python${{ matrix.python_version }} -m pip install xdsl~=0.55.0 xdsl-jax==0.1.1
+        # Install graphviz for testing the mlir-op-graph integration
+        sudo apt-get install -y graphviz
+        python${{ matrix.python_version }} -m pip install graphviz
+
+    - name: Run Unified Compiler Tests
+      run: |
+        python${{ matrix.python_version }} -m pytest frontend/test/pytest -n auto -m "xdsl"
+
+

.github/workflows/build-wheel-linux-x86_64.yaml

@@ -520,3 +520,14 @@ jobs:
       if: env.BUILD_STANDALONE_PLUGIN == 'true'
       run: |
         python${{ matrix.python_version }} -m pytest standalone_plugin_wheel/standalone_plugin/test -n auto
+
+    - name: Install Unified Compiler Dependencies
+      run: |
+        python${{ matrix.python_version }} -m pip install xdsl~=0.55.0 xdsl-jax==0.1.1
+        # Install graphviz for testing the mlir-op-graph integration
+        sudo apt-get install -y graphviz
+        python${{ matrix.python_version }} -m pip install graphviz
+
+    - name: Run Unified Compiler Tests
+      run: |
+        python${{ matrix.python_version }} -m pytest frontend/test/pytest -n auto -m "xdsl"

.github/workflows/build-wheel-macos-arm64.yaml

@@ -501,3 +501,14 @@ jobs:
       if: env.BUILD_STANDALONE_PLUGIN == 'true'
       run: |
         python${{ matrix.python_version }} -m pytest standalone_plugin_wheel/standalone_plugin/test -n auto
+
+    - name: Install Unified Compiler Dependencies
+      run: |
+        python${{ matrix.python_version }} -m pip install xdsl~=0.55.0 xdsl-jax==0.1.1
+        # Install graphviz for testing the mlir-op-graph integration
+        brew install graphviz
+        python${{ matrix.python_version }} -m pip install graphviz
+
+    - name: Run Unified Compiler Tests
+      run: |
+        python${{ matrix.python_version }} -m pytest frontend/test/pytest -n auto -m "xdsl"

.github/workflows/check-catalyst.yaml

@@ -533,6 +533,17 @@ jobs:
         MDD_BENCHMARK_PRECISION=1 \
         python3 -m pytest demos -k "tutorial_qft_arithmetics.ipynb" --nbmake
 
+    - name: Install Unified Compiler Dependencies
+      run: |
+        python3 -m pip install xdsl~=0.55.0 xdsl-jax==0.1.1
+        # Install graphviz for testing the mlir-op-graph integration
+        sudo apt-get install -y graphviz
+        python3 -m pip install graphviz
+
+    - name: Run Unified Compiler Tests
+      run: |
+        python3 -m pytest frontend/test/pytest -n auto -m "xdsl"
+
   frontend-tests-lightning-kokkos:
     name: Frontend Tests (backend="lightning.kokkos")
     needs: [constants, llvm, runtime, quantum, determine_runner]

.github/workflows/check-pl-compat.yaml

@@ -207,3 +207,36 @@ jobs:
         # We skip them for now. These demos should be properly moved to the qml repo.
         MDD_BENCHMARK_PRECISION=1 \
         python3 -m pytest demos -k "tutorial_qft_arithmetics.ipynb" --nbmake
+
+    - name: Check If Unified Compiler Tests Should Execute
+      id: check_unified_compiler
+      run: |
+        # Check if installed pennylane has python_compiler module
+        EXIT_CODE=0
+        python3 -c "from pennylane.compiler import python_compiler" || EXIT_CODE=$?
+
+        # For Unified Compiler tests to execute, PennyLane must NOT have the 'compiler.python_compiler'
+        # submodule, and Catalyst MUST have the 'python_interface' submodule
+        if [[ -d "$GITHUB_WORKSPACE/frontend/catalyst/python_interface" && EXIT_CODE -ne 0 ]]; then
+          echo "unified_compiler_available='true'" >> $GITHUB_OUTPUT
+        else
+          echo "unified_compiler_available='false'" >> $GITHUB_OUTPUT
+        fi
+
+    - name: Install Unified Compiler Dependencies
+      if: steps.check_unified_compiler.outputs.unified_compiler_available == 'true'
+      run: |
+        python3 -m pip install xdsl~=0.55.0 xdsl-jax==0.1.1
+        # Install graphviz for testing the mlir-op-graph integration
+        sudo apt-get install -y graphviz
+        python3 -m pip install graphviz
+
+    - name: Run Unified Compiler Tests
+      if: steps.check_unified_compiler.outputs.unified_compiler_available == 'true'
+      run: |
+        python3 -m pytest frontend/test/pytest -n auto -m "xdsl"
+
+    - name: Run Unified Compiler Tests (Kokkos)
+      if: steps.check_unified_compiler.outputs.unified_compiler_available == 'true'
+      run: |
+        python3 -m pytest frontend/test/pytest --backend="lightning.kokkos" -n auto -m "xdsl"

doc/releases/changelog-dev.md

@@ -2,6 +2,98 @@
 
 <h3>New features since last release</h3>
 
+* Catalyst now features a unified compilation framework, which enables users and developers to design
+  and implement compilation passes in Python in addition to C++, on the same Catalyst IR. The Python
+  interface relies on the xDSL library to represent and manipulate programs (analogous to the MLIR library
+  in C++). As a result, transformations can be quickly prototyped, easily debugged, and dynamically integrated
+  into Catalyst without changes to the compiled Catalyst package. The new module is available under the
+  `catalyst.python_interface` namespace.
+  [(#2199)](https://github.com/PennyLaneAI/catalyst/pull/2199)
+
+  This functionality was originally developed as part of the PennyLane package, and has been migrated here.
+  For earlier development notes to the feature, please refer to the
+  [PennyLane release notes](https://docs.pennylane.ai/en/stable/development/release_notes.html#release-0-43-0).
+  In addition, the following changes have been made since the last release:
+
+  * Add an experimental `outline_state_evolution_pass` xDSL pass to `catalyst.python_interface.transforms`,
+    which moves all quantum gate operations to a private callable.
+    [(#8367)](https://github.com/PennyLaneAI/pennylane/pull/8367)
+
+  * A new experimental `split_non_commuting_pass` compiler pass has been added to
+    `catalyst.python_interface.transforms`. This pass splits quantum functions that
+    measure observables on the same wires into multiple function executions, where
+    each execution measures observables on different wires (using the "wires" grouping
+    strategy). The original function is replaced with calls to these generated functions,
+    and the results are combined appropriately.
+    [(#8531)](https://github.com/PennyLaneAI/pennylane/pull/8531)
+
+  * Add the `PCPhaseOp` operation to the xDSL Quantum dialect.
+    [(#8621)](https://github.com/PennyLaneAI/pennylane/pull/8621)
+
+  * Users can now apply xDSL passes without the need to pass the `pass_plugins` argument to
+    the `qjit` decorator.
+    [(#8572)](https://github.com/PennyLaneAI/pennylane/pull/8572)
+    [(#8573)](https://github.com/PennyLaneAI/pennylane/pull/8573)
+    [(#2169)](https://github.com/PennyLaneAI/catalyst/pull/2169)
+    [(#2183)](https://github.com/PennyLaneAI/catalyst/pull/2183)
+
+  * The :meth:`catalyst.python_interface.transforms.convert_to_mbqc_formalism_pass` now
+    supports :class:`~xdsl.dialects.scf.IndexSwitchOp` in IR and ignores regions that have no body.
+    [(#8632)](https://github.com/PennyLaneAI/pennylane/pull/8632)
+
+  * The `convert_to_mbqc_formalism` compilation pass now outlines the operations to represent a gate
+    in the MBQC formalism into subroutines in order to reduce the IR size for large programs.
+    [(#8619)](https://github.com/PennyLaneAI/pennylane/pull/8619)
+
+  * The :meth:`catalyst.python_interface.Compiler.run` method now accepts a string as input,
+    which is parsed and transformed with xDSL.
+    [(#8587)](https://github.com/PennyLaneAI/pennylane/pull/8587)
+
+  * An `is_xdsl_pass` function has been added to the `catalyst.python_interface.pass_api` module.
+    This function checks if a pass name corresponds to an xDSL implemented pass.
+    [(#8572)](https://github.com/PennyLaneAI/pennylane/pull/8572)
+
+  * A new `catalyst.python_interface.utils` submodule has been added, containing general-purpose utilities for
+    working with xDSL. This includes a function that extracts the concrete value of scalar, constant SSA values.
+    [(#8514)](https://github.com/PennyLaneAI/pennylane/pull/8514)
+
+  * The `catalyst.python_interface.visualization` module has been renamed to
+    `catalyst.python_interface.inspection`, and various utility functions within this module
+    have been streamlined.
+    [(#2237)](https://github.com/PennyLaneAI/catalyst/pull/2237)
+
+  * The experimental xDSL :func:`~catalyst.python_interface.transforms.measurements_from_samples_pass`
+    pass has been updated to support `shots` defined by an `arith.constant` operation.
+    [(#8460)](https://github.com/PennyLaneAI/pennylane/pull/8460)
+
+  * The experimental xDSL :func:`~catalyst.python_interface.transforms.diagonalize_measurements`
+    pass has been updated to fix a bug that included the wrong SSA value for final qubit insertion
+    and deallocation at the end of the circuit. A clear error is now also raised when there are
+    observables with overlapping wires.
+    [(#8383)](https://github.com/PennyLaneAI/pennylane/pull/8383)
+
+  * Fixes a bug in the constructor of the xDSL Quantum dialect's `QubitUnitaryOp` that
+    prevented an instance from being constructed.
+    [(#8456)](https://github.com/PennyLaneAI/pennylane/pull/8456)
+
+  * Removed the `catalyst.python_interface.dialects.transform` module in favor of
+    using the `xdsl.dialects.transform` module directly.
+    [(#2261)](https://github.com/PennyLaneAI/catalyst/pull/2261)
+
+  * Added a "Unified Compiler Cookbook" RST file, along with tutorials, to `catalyst.python_interface.doc`,
+    which provides a quickstart guide for getting started with xDSL and its integration with PennyLane and
+    Catalyst.
+    [(#8571)](https://github.com/PennyLaneAI/pennylane/pull/8571)
+
+  * xDSL passes are now automatically detected when using the `qjit` decorator.
+    This removes the need to pass the `pass_plugins` argument to the `qjit` decorator.
+    [(#2169)](https://github.com/PennyLaneAI/catalyst/pull/2169)
+    [(#2183)](https://github.com/PennyLaneAI/catalyst/pull/2183)
+
+  * The ``mlir_opt`` property now correctly handles xDSL passes by automatically
+    detecting when the Python compiler is being used and routing through it appropriately.
+    [(#2190)](https://github.com/PennyLaneAI/catalyst/pull/2190)
+
 * RTIO dialect is added to bypass the compilation flow from OpenAPL to ARTIQ’s LLVM IR. It is introduced to bridge the gap between ION dialect and ARTIQ’s LLVM IR. The design philosophy of RTIO dialect is primarily event-based. Every operation is asynchronous; sync behaviour occurs only via `rtio.sync` or `wait operand` in event operation.
   [(#2185)](https://github.com/PennyLaneAI/catalyst/pull/2185)
 
@@ -127,15 +219,6 @@
 * `qml.grad` and `qml.jacobian` can now be used with `qjit` when program capture is enabled.
   [(#2078)](https://github.com/PennyLaneAI/catalyst/pull/2078)
 
-* xDSL passes are now automatically detected when using the `qjit` decorator.
-  This removes the need to pass the `pass_plugins` argument to the `qjit` decorator.
-  [(#2169)](https://github.com/PennyLaneAI/catalyst/pull/2169)
-  [(#2183)](https://github.com/PennyLaneAI/catalyst/pull/2183)
-
-* The ``mlir_opt`` property now correctly handles xDSL passes by automatically
-  detecting when the Python compiler is being used and routing through it appropriately.
-  [(#2190)](https://github.com/PennyLaneAI/catalyst/pull/2190)
-
 * Dynamically allocated wires can now be passed into control flow and subroutines.
   [(#2130)](https://github.com/PennyLaneAI/catalyst/pull/2130)
 
@@ -340,11 +423,9 @@
   [(#2245)](https://github.com/PennyLaneAI/catalyst/pull/2245)
   [(#2254)](https://github.com/PennyLaneAI/catalyst/pull/2254)
 
-
   * Refactor QEC tablegen files to separate QEC operations into a new `QECOp.td` file
   [(#2253](https://github.com/PennyLaneAI/catalyst/pull/2253)
 
-
   * Removed the `getRotationKind` and `setRotationKind` methods from
   the QEC interface `QECOpInterface` to simplify the interface.
   [(#2250)](https://github.com/PennyLaneAI/catalyst/pull/2250)

frontend/catalyst/compiler.py

@@ -384,13 +384,13 @@ def to_mlir_opt(
     *args, stdin=None, options: Optional[CompileOptions] = None, using_python_compiler=False
 ):
     """echo ${input} | catalyst --tool=opt *args *opts -"""
-    # Check if we need to use Python compiler for xDSL passes
+    # Check if we need to use the Python interface for xDSL passes
     if using_python_compiler:
-        # Use Python compiler path for xDSL passes
+        # Use the Python interface path for xDSL passes
         # pylint: disable-next=import-outside-toplevel
-        from pennylane.compiler.python_compiler import Compiler as PythonCompiler
+        from catalyst.python_interface import Compiler as UnifiedCompiler
 
-        compiler = PythonCompiler()
+        compiler = UnifiedCompiler()
         stdin = compiler.run(stdin, callback=None)
 
     # These are the options that may affect compilation
@@ -548,7 +548,7 @@ def check_nested_operations(op):
 
     @debug_logger
     def is_using_python_compiler(self, mlir_module=None):
-        """Returns true if we need the Python compiler path.
+        """Returns true if we need the Python interface path.
 
         This happens when:
         1. xDSL plugin is explicitly loaded (legacy), OR
@@ -594,7 +594,8 @@ def _create_xdsl_pass_save_callback(self, workspace):
             workspace: The workspace directory path
 
         Returns:
-            Callable or None: The callback function if intermediate saving is enabled, None otherwise
+            Callable or None: The callback function if intermediate saving is enabled, None
+            otherwise
         """
         if not (workspace and self.options.keep_intermediate >= KeepIntermediateLevel.CHANGED):
             return None
@@ -606,7 +607,7 @@ def _create_xdsl_pass_save_callback(self, workspace):
         os.makedirs(user_transform_dir, exist_ok=True)
 
         class SavePassIRCallback:
-            """Callback to save IR after each pass in python_compiler."""
+            """Callback to save IR after each pass in python_interface."""
 
             def __init__(self, transform_dir):
                 self.transform_dir = transform_dir
@@ -666,10 +667,10 @@ def run(self, mlir_module, *args, **kwargs):
             # We keep this module here to keep xDSL requirement optional
             # Only move this is it has been decided that xDSL is no longer optional.
             # pylint: disable-next=import-outside-toplevel
-            from pennylane.compiler.python_compiler import Compiler as PythonCompiler
+            from catalyst.python_interface import Compiler as UnifiedCompiler
 
             callback = self._create_xdsl_pass_save_callback(workspace)
-            compiler = PythonCompiler()
+            compiler = UnifiedCompiler()
             ir = compiler.run(ir, callback=callback)
 
         return self.run_from_ir(

frontend/catalyst/jax_primitives_utils.py

@@ -398,9 +398,7 @@ def transform_named_sequence_lowering(jax_ctx: mlir.LoweringRuleContext, pipelin
 
                 try:
                     # pylint: disable=import-outside-toplevel
-                    from pennylane.compiler.python_compiler.pass_api import (
-                        is_xdsl_pass,
-                    )
+                    from catalyst.python_interface.pass_api import is_xdsl_pass
 
                     if is_xdsl_pass(name):
                         uses_xdsl_passes = True

frontend/catalyst/python_interface/__init__.py

@@ -0,0 +1,26 @@
+# Copyright 2025 Xanadu Quantum Technologies Inc.
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+#     http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Unified Compiler API for integration of Catalyst with xDSL."""
+
+from .compiler import Compiler
+from .inspection import QMLCollector
+from .parser import QuantumParser
+from .pass_api import compiler_transform
+
+__all__ = [
+    "Compiler",
+    "compiler_transform",
+    "QuantumParser",
+    "QMLCollector",
+]