Merge pull request #5135 from YosysHQ/emil/ast-ownership

ast, read_verilog: ownership in AST, use C++ styles for parser and lexer

Author: widlarizer

.github/actions/setup-build-env/action.yml

@@ -8,7 +8,7 @@ runs:
       shell: bash
       run: |
         sudo apt-get update
-        sudo apt-get install gperf build-essential bison flex libreadline-dev gawk tcl-dev libffi-dev git graphviz xdot pkg-config python3 libboost-system-dev libboost-python-dev libboost-filesystem-dev zlib1g-dev libbz2-dev
+        sudo apt-get install gperf build-essential bison flex libfl-dev libreadline-dev gawk tcl-dev libffi-dev git graphviz xdot pkg-config python3 libboost-system-dev libboost-python-dev libboost-filesystem-dev zlib1g-dev libbz2-dev
 
     - name: Install macOS Dependencies
       if: runner.os == 'macOS'

.github/workflows/extra-builds.yml

@@ -27,19 +27,20 @@ jobs:
         with:
           submodules: true
           persist-credentials: false
+      - run: sudo apt-get install libfl-dev
       - name: Build
         run: make vcxsrc YOSYS_VER=latest
       - uses: actions/upload-artifact@v4
         with:
           name: vcxsrc
           path: yosys-win32-vcxsrc-latest.zip
-  
+
   vs-build:
     name: Visual Studio build
     runs-on: windows-latest
     needs: [vs-prep, pre_job]
     if: needs.pre_job.outputs.should_skip != 'true'
-    steps:  
+    steps:
       - uses: actions/download-artifact@v4
         with:
           name: vcxsrc
@@ -68,9 +69,20 @@ jobs:
           WASI_SDK_URL=https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-19/wasi-sdk-19.0-linux.tar.gz
           if ! [ -d ${WASI_SDK} ]; then curl -L ${WASI_SDK_URL} | tar xzf -; fi
 
+          FLEX_VER=2.6.4
+          FLEX=flex-${FLEX_VER}
+          FLEX_URL=https://github.com/westes/flex/releases/download/v${FLEX_VER}/${FLEX}.tar.gz
+          if ! [ -d ${FLEX} ]; then curl -L ${FLEX_URL} | tar xzf -; fi
+
+          mkdir -p flex-build
+          (cd flex-build &&
+            ../${FLEX}/configure --prefix=$(pwd)/../flex-prefix &&
+            make &&
+            make install)
+
           mkdir -p build
           cat > build/Makefile.conf <<END
-          export PATH := $(pwd)/${WASI_SDK}/bin:${PATH}
+          export PATH := $(pwd)/${WASI_SDK}/bin:$(pwd)/flex-prefix/bin:${PATH}
           WASI_SYSROOT := $(pwd)/${WASI_SDK}/share/wasi-sysroot
 
           CONFIG := wasi
@@ -80,6 +92,8 @@ jobs:
           ENABLE_READLINE := 0
           ENABLE_PLUGINS := 0
           ENABLE_ZLIB := 0
+
+          CXXFLAGS += -I$(pwd)/flex-prefix/include
           END
 
           make -C build -f ../Makefile CXX=clang -j$(nproc)

Makefile

@@ -132,8 +132,8 @@ ifeq ($(ENABLE_PYOSYS),1)
 CXXFLAGS += -I$(BREW_PREFIX)/boost/include
 LINKFLAGS += -L$(BREW_PREFIX)/boost/lib -L$(BREW_PREFIX)/boost-python3/lib
 endif
-CXXFLAGS += -I$(BREW_PREFIX)/readline/include
-LINKFLAGS += -L$(BREW_PREFIX)/readline/lib
+CXXFLAGS += -I$(BREW_PREFIX)/readline/include -I$(BREW_PREFIX)/flex/include
+LINKFLAGS += -L$(BREW_PREFIX)/readline/lib -L$(BREW_PREFIX)/flex/lib
 PKG_CONFIG_PATH := $(BREW_PREFIX)/libffi/lib/pkgconfig:$(PKG_CONFIG_PATH)
 PKG_CONFIG_PATH := $(BREW_PREFIX)/tcl-tk/lib/pkgconfig:$(PKG_CONFIG_PATH)
 export PATH := $(BREW_PREFIX)/bison/bin:$(BREW_PREFIX)/gettext/bin:$(BREW_PREFIX)/flex/bin:$(PATH)

README.md

@@ -80,10 +80,10 @@ recommended) and some standard tools such as GNU Flex, GNU Bison, and GNU Make.
 TCL, readline and libffi are optional (see ``ENABLE_*`` settings in Makefile).
 Xdot (graphviz) is used by the ``show`` command in yosys to display schematics.
 
-For example on Ubuntu Linux 16.04 LTS the following commands will install all
+For example on Ubuntu Linux 22.04 LTS the following commands will install all
 prerequisites for building yosys:
 
-	$ sudo apt-get install build-essential clang lld bison flex \
+	$ sudo apt-get install build-essential clang lld bison flex libfl-dev \
 		libreadline-dev gawk tcl-dev libffi-dev git \
 		graphviz xdot pkg-config python3 libboost-system-dev \
 		libboost-python-dev libboost-filesystem-dev zlib1g-dev

docs/source/getting_started/installation.rst

@@ -88,18 +88,18 @@ Build prerequisites
 ^^^^^^^^^^^^^^^^^^^
 
 A C++ compiler with C++17 support is required as well as some standard tools
-such as GNU Flex, GNU Bison, Make and Python.  Some additional tools: readline,
-libffi, Tcl and zlib; are optional but enabled by default (see
+such as GNU Flex, GNU Bison (>=3.8), Make, and Python (>=3.11). Some additional
+tools: readline, libffi, Tcl and zlib; are optional but enabled by default (see
 :makevar:`ENABLE_*` settings in Makefile). Graphviz and Xdot are used by the
 `show` command to display schematics.
 
-Installing all prerequisites for Ubuntu 20.04:
+Installing all prerequisites for Ubuntu 22.04:
 
 .. code:: console
 
-   sudo apt-get install gperf build-essential bison flex \
-      libreadline-dev gawk tcl-dev libffi-dev git graphviz \
-      xdot pkg-config python3 libboost-system-dev \
+   sudo apt-get install gperf build-essential clang lld bison flex libfl-dev \
+      libreadline-dev gawk tcl-dev libffi-dev git \
+      graphviz xdot pkg-config python3 libboost-system-dev \
       libboost-python-dev libboost-filesystem-dev zlib1g-dev
 
 Installing all prerequisites for macOS 13 (with Homebrew):
@@ -137,8 +137,9 @@ For Cygwin use the following command to install all prerequisites, or select the
    minimum required version of Python is 3.11.  This means that Cygwin is not
    compatible with many of the Python-based frontends.  While this does not
    currently prevent Yosys itself from working, no guarantees are made for
-   continued support.  It is instead recommended to use Windows Subsystem for
-   Linux (WSL) and follow the instructions for Ubuntu.
+   continued support.  You may also need to specify `CXXSTD=gnu++17` to resolve
+   missing `strdup` function when using gcc.  It is instead recommended to use
+   Windows Subsystem for Linux (WSL) and follow the instructions for Ubuntu.
 
 .. 
    For MSYS2 (MINGW64):

docs/source/yosys_internals/flow/verilog_frontend.rst

@@ -47,9 +47,9 @@ be found in :file:`frontends/verilog/verilog_lexer.l` in the Yosys source tree.
 The lexer does little more than identifying all keywords and literals recognised
 by the Yosys Verilog frontend.
 
-The lexer keeps track of the current location in the Verilog source code using
-some global variables. These variables are used by the constructor of AST nodes
-to annotate each node with the source code location it originated from.
+The lexer keeps track of the current location in the Verilog source code with
+a ``VerilogLexer::out_loc`` and uses it to construct parser-defined
+symbol objects.
 
 Finally the lexer identifies and handles special comments such as "``// synopsys
 translate_off``" and "``// synopsys full_case``". (It is recommended to use
@@ -178,21 +178,22 @@ properties:
 
 -  | Source code location
    | Each ``AST::AstNode`` is automatically annotated with the current source
-     code location by the ``AST::AstNode`` constructor. It is stored in the
-     ``std::string filename`` and ``int linenum`` member variables.
+     code location by the ``AST::AstNode`` constructor. The ``location`` type
+     is a manual reimplementation of the bison-provided location type. This
+     type is defined at ``frontends/verilog/verilog_location.h``.
 
-The ``AST::AstNode`` constructor can be called with up to two child nodes that
-are automatically added to the list of child nodes for the new object. This
+The ``AST::AstNode`` constructor can be called with up to 4 child nodes. This
 simplifies the creation of AST nodes for simple expressions a bit. For example
 the bison code for parsing multiplications:
 
 .. code:: none
-   	:number-lines:
+   :number-lines:
 
-	basic_expr '*' attr basic_expr {
-		$$ = new AstNode(AST_MUL, $1, $4);
-		append_attr($$, $3);
-	} |
+   basic_expr TOK_ASTER attr basic_expr {
+     $$ = std::make_unique<AstNode>(AST_MUL, std::move($1), std::move($4));
+     SET_AST_NODE_LOC($$.get(), @1, @4);
+     append_attr($$.get(), $3);
+   } |
 
 The generated AST data structure is then passed directly to the AST frontend
 that performs the actual conversion to RTLIL.
@@ -204,7 +205,7 @@ tree respectively.
 Transforming AST to RTLIL
 -------------------------
 
-The AST Frontend converts a set of modules in AST representation to modules in
+The AST frontend converts a set of modules in AST representation to modules in
 RTLIL representation and adds them to the current design. This is done in two
 steps: simplification and RTLIL generation.