[ENH] Use hatch (#1553)

Co-authored-by: Martin Durant <[email protected]>

Author: fleming79

.github/workflows/codespell.yml

@@ -14,6 +14,6 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Codespell
         uses: codespell-project/actions-codespell@v1

.github/workflows/main.yaml

@@ -2,9 +2,9 @@ name: CI
 
 on:
   push:
-    branches: "*"
+    branches: ["*"]
   pull_request:
-    branches: master
+    branches: [master]
 
 jobs:
   linux:
@@ -13,14 +13,14 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        PY: ["3.8", "3.9", "3.10", "3.11"]
+        PY: ["3.8", "3.9", "3.10", "3.11", "3.12"]
 
     env:
       CIRUN: true
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
 
@@ -34,20 +34,22 @@ jobs:
       - name: Run Tests
         shell: bash -l {0}
         run: |
+          pip install s3fs
+          pip uninstall s3fs
+          pip install -e .[test_full]
+          pip install s3fs --no-deps
           pytest -v
 
   win:
     name: pytest-win
     runs-on: windows-2019
-    strategy:
-      fail-fast: false
 
     env:
       CIRUN: true
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
 
@@ -59,6 +61,10 @@ jobs:
       - name: Run Tests
         shell: bash -l {0}
         run: |
+          pip install s3fs
+          pip uninstall s3fs
+          pip install -e .[test]
+          pip install s3fs --no-deps
           pytest -v
 
   lint:
@@ -75,7 +81,7 @@ jobs:
 #    runs-on: ubuntu-latest
 #    steps:
 #      - name: Checkout
-#        uses: actions/checkout@v3
+#        uses: actions/checkout@v4
 #
 #      - name: Setup conda
 #        uses: mamba-org/setup-micromamba@v1
@@ -93,7 +99,7 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
 
@@ -105,35 +111,20 @@ jobs:
       - name: Local install
         shell: bash -l {0}
         run: |
-          git config --global user.email "[email protected]"
-          git config --global user.name "Your Name"
-          git tag -a 3000 -m "fake"
-          pip install -e .
-
-      - name: Clone s3fs
-        shell: bash -l {0}
-        run: git clone https://github.com/fsspec/s3fs
-
-      - name: Install s3fs
-        shell: bash -l {0}
-        run: |
-          pip install -e ./s3fs --no-deps
+          sh install_s3fs.sh
+          pip install -e .[test,test_downstream]
+          pip list
 
       - name: Run fsspec tests
         shell: bash -l {0}
         run: |
           pytest -v fsspec/tests/test_downstream.py
 
-      - name: clone dask
-        shell: bash -l {0}
-        run: |
-          git clone https://github.com/dask/dask
-          pip install -e ./dask
-
       - name: Run dask tests
         shell: bash -l {0}
         run: |
-          pytest -v dask/dask/bytes
+          dask_test_path=$(python -c "import dask.bytes;print(dask.bytes.__path__[0])")
+          pytest -v $dask_test_path
 
   fsspec_friends:
     name: ${{ matrix.FRIEND }}-pytest
@@ -151,7 +142,7 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Setup conda
         uses: mamba-org/setup-micromamba@v1

.github/workflows/pypipublish.yaml

@@ -8,19 +8,19 @@ jobs:
   deploy:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
           python-version: "3.x"
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install setuptools setuptools-scm wheel twine
+          pip install hatch twine
       - name: Build and publish
         env:
           TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
           TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
         run: |
-          python setup.py sdist bdist_wheel
+          hatch build
           twine upload dist/*

.gitignore

@@ -28,7 +28,7 @@ wheels/
 .installed.cfg
 *.egg
 pip-wheel-metadata/
-
+_version.py
 # PyInstaller
 #  Usually these files are written by a python script from a template
 #  before PyInstaller builds the exe, so as to inject date/other infos into it.
@@ -121,3 +121,4 @@ ENV/
 *.swp
 
 build/
+downstream/*

.pre-commit-config.yaml

@@ -5,33 +5,19 @@ exclude: >
 repos:
 
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v3.4.0
+    rev: v4.5.0
     hooks:
       - id: trailing-whitespace
       - id: end-of-file-fixer
       - id: check-docstring-first
       - id: check-json
       - id: check-yaml
-  - repo: https://github.com/ambv/black
-    rev: 22.3.0
-    hooks:
-      - id: black
   - repo: https://github.com/astral-sh/ruff-pre-commit
     # Ruff version.
-    rev: v0.2.1
+    rev: v0.3.4
     hooks:
       # Run the linter.
       - id: ruff
-        args: [ --fix ]
-  - repo: https://github.com/PyCQA/flake8
-    rev: 5.0.4
-    hooks:
-      - id: flake8
-  - repo: https://github.com/asottile/seed-isort-config
-    rev: v2.2.0
-    hooks:
-      - id: seed-isort-config
-  - repo: https://github.com/pre-commit/mirrors-isort
-    rev: v5.7.0
-    hooks:
-      - id: isort
+        args: [ --fix, "--show-fixes"]
+      - id: ruff-format
+        types_or: [python]

MANIFEST.in

@@ -46,6 +46,27 @@ in the "ci/" directory. Note that the main environment is called "py38",
 but it is expected that the version of python installed be adjustable at
 CI runtime. For local use, pick a version suitable for you.
 
+```bash
+# For a new environment (mamba / conda).
+mamba create -n fsspec -c conda-forge  python=3.9 -y
+conda activate fsspec
+
+# Standard dev test install.
+pip install -e ".[dev,test]"
+
+# Full tests except for downstream
+pip install s3fs
+pip uninstall s3fs
+pip install -e .[dev,test_full]
+pip install s3fs --no-deps
+pytest -v
+
+# Downstream tests.
+sh install_s3fs.sh
+# Windows powershell.
+install_s3fs.sh
+```
+
 ### Testing
 
 Tests can be run in the dev environment, if activated, via ``pytest fsspec``.

README.md

@@ -3,18 +3,5 @@ channels:
   - conda-forge
 dependencies:
   - python=3.9
-  - dask
-  - pandas
-  - xarray
-  - zarr
-  - pytest<8
-  - pytest-cov
-  - pytest-rerunfailures
-  - pytest-timeout
-  - pytest-xdist
-  - s3fs
-  - requests
-  - moto <5
-  - sqlalchemy<2
-  - flask
-  - dask-expr
+  - pip:
+      - git+https://github.com/dask/dask

ci/environment-downstream.yml

@@ -8,7 +8,6 @@ dependencies:
   - pytest-benchmark
   - pytest-cov
   - pytest-mock
-  - pytest-vcr
   - pip
   - pytest<8
   - ujson

ci/environment-friends.yml

@@ -2,41 +2,9 @@ name: test_env
 channels:
   - conda-forge
 dependencies:
-  # - python=3.8  # set by env
   - pip
-  - paramiko
-  - requests
-  - zstandard
-  - python-snappy
-  - aiohttp
-  - lz4
-  - distributed
-  - dask
-  - pyarrow
-  - panel
-  - notebook
-  - pygit2
   - git
-  - s3fs
-  - pyftpdlib
-  - cloudpickle
-  - pytest <8
-  - pytest-asyncio !=0.22.0
-  - pytest-benchmark
-  - pytest-cov
-  - pytest-mock
-  - pytest-vcr
   - py
-  - fusepy
-  - fastparquet
-  - tomli < 2
-  - msgpack-python
-  - python-libarchive-c
-  - numpy
-  - nomkl
-  - jinja2
-  - tqdm
-  - urllib3 <=1.26.18
   - pip:
     - hadoop-test-cluster
     - smbprotocol