update JuliaSyntax to v1 (#121)

* update JuliaSyntax to v1

* fix bug and add test

* fix random typos

* fix test setup

* fix deprecation

Author: ericphanson

Project.toml

@@ -1,7 +1,7 @@
 name = "PackageAnalyzer"
 uuid = "e713c705-17e4-4cec-abe0-95bf5bf3e10c"
 authors = ["Mosè Giordano <[email protected]>"]
-version = "3.1.0"
+version = "3.2.0"
 
 [deps]
 AbstractTrees = "1520ce14-60c1-5f80-bbc7-55ef81b5835c"
@@ -28,7 +28,7 @@ Git = "1.2.1"
 GitHub = "5.4"
 Legolas = "0.5"
 JSON3 = "1.5.1"
-JuliaSyntax = "0.4"
+JuliaSyntax = "1"
 LicenseCheck = "0.2"
 RegistryInstances = "0.1"
 julia = "1.10"

README.md

@@ -54,6 +54,8 @@ Package Flux:
 
 See the [docs](https://JuliaEcosystem.github.io/PackageAnalyzer.jl/dev/) for more!
 
+One special feature of PackageAnalyzer is a custom JuliaSyntax-based line-of-code counting implementation which correctly distinguishes between Julia docstrings, comments, and code.
+
 ## Talks and blog posts
 
 Check out our JuliaCon 2023 talk:

src/LineCategories.jl

@@ -3,7 +3,7 @@
 module CategorizeLines
 export LineCategories, LineCategory, Blank, Code, Docstring, Comment, categorize_lines!
 
-using JuliaSyntax: haschildren, children, SourceFile, kind, @K_str, source_line
+using JuliaSyntax: JuliaSyntax, children, SourceFile, kind, @K_str, source_line
 
 # Every line will have a single category. This way the total number across all categories
 # equals the total number of lines. This is useful for debugging and is reassuring to users.
@@ -32,7 +32,7 @@ Currently:
 @enum LineCategory Blank Code Comment Docstring
 
 # We will store the categories assigned to each line in a file with the following structure.
-# This keeps the `SourceFile` to facillitate printing.
+# This keeps the `SourceFile` to facilitate printing.
 """
     LineCategories(path)
 
@@ -84,11 +84,11 @@ function categorize_lines!(d::LineCategories, node, source, nesting=0, pos=1, pa
     k = kind(node)
 
     # Recurse over children
-    is_leaf = !haschildren(node)
+    is_leaf = JuliaSyntax.is_leaf(node)
     if !is_leaf
         new_nesting = nesting + 1
         p = pos
-        for x in children(node)
+        for x in something(children(node), ())
             categorize_lines!(d, x, source, new_nesting, p, k)
             p += x.span
         end

src/analyze.jl

@@ -41,8 +41,8 @@ PackageV1 Pluto:
   * tree hash: db1306745717d127037c5697436b04cfb9d7b3dd
   * Julia code in `src`: 8337 lines
   * Julia code in `test`: 5448 lines (39.5% of `test` + `src`)
-  * documention in `docs`: 0 lines (0.0% of `docs` + `src`)
-  * documention in README: 118 lines
+  * documentation in `docs`: 0 lines (0.0% of `docs` + `src`)
+  * documentation in README: 118 lines
   * has license(s) in file: MIT
     * filename: LICENSE
     * OSI approved: true
@@ -102,8 +102,8 @@ PackageV1 DataFrames:
   * tree hash: db2a9cb664fcea7836da4b414c3278d71dd602d2
   * Julia code in `src`: 15628 lines
   * Julia code in `test`: 21089 lines (57.4% of `test` + `src`)
-  * documention in `docs`: 6270 lines (28.6% of `docs` + `src`)
-  * documention in README: 21 lines
+  * documentation in `docs`: 6270 lines (28.6% of `docs` + `src`)
+  * documentation in README: 21 lines
   * has license(s) in file: MIT
     * filename: LICENSE.md
     * OSI approved: true
@@ -138,7 +138,7 @@ end
     analyze_manifest([path_to_manifest]; registries=reachable_registries(),
                      auth=github_auth(), sleep=0)
 
-Convienence function to run [`find_packages_in_manifest`](@ref) then [`analyze`](@ref) on the results. Positional argument `path_to_manifest` defaults to `joinpath(dirname(Base.active_project()), "Manifest.toml")`.
+Convenience function to run [`find_packages_in_manifest`](@ref) then [`analyze`](@ref) on the results. Positional argument `path_to_manifest` defaults to `joinpath(dirname(Base.active_project()), "Manifest.toml")`.
 """
 function analyze_manifest(args...; registries=reachable_registries(),
     auth=github_auth(), sleep=0)

src/count_loc.jl

@@ -94,14 +94,16 @@ struct GreenNodeWrapper
 end
 
 function AbstractTrees.children(wrapper::GreenNodeWrapper)
-    return map(n -> GreenNodeWrapper(n, wrapper.source), JuliaSyntax.children(wrapper.node))
+    return map(n -> GreenNodeWrapper(n, wrapper.source), something(JuliaSyntax.children(wrapper.node), ()))
 end
 
+const EMPTY_TREE = JuliaSyntax.GreenNode(JuliaSyntax.SyntaxHead(K"toplevel", 0), 0, JuliaSyntax.GreenNode{JuliaSyntax.SyntaxHead}[])
+
 function parse_green_node(file_path)
     file = read(file_path, String)
     parsed = @maybecatch(JuliaSyntax.parseall(JuliaSyntax.GreenNode, file; ignore_trivia=false, filename=file_path),
         "Error during `JuliaSyntax.parseall` of $(file_path)",
-        JuliaSyntax.GreenNode(JuliaSyntax.SyntaxHead(K"toplevel", 0), 0, ()))
+        EMPTY_TREE)
     return GreenNodeWrapper(parsed, JuliaSyntax.SourceFile(file; filename=file_path))
 end
 
@@ -127,7 +129,7 @@ function count_julia_lines_of_code(dir)
     table = LinesOfCodeV2[]
     for path in readdir(dir; join=true)
         n_files = 0
-        # Could be a LittleDict for efficency, but that would require
+        # Could be a LittleDict for efficiency, but that would require
         # a dependency on OrderedCollections.jl
         counts = Dict{LineCategory,Int}(Comment => 0,
             Blank => 0,

src/utilities.jl

@@ -5,7 +5,7 @@
 """
     PackageAnalyzer.github_auth(token::String="")
 
-Obtain a GitHub authetication.  Use the `token` argument if it is non-empty,
+Obtain a GitHub authentication.  Use the `token` argument if it is non-empty,
 otherwise use the `GITHUB_TOKEN` and `GITHUB_AUTH` environment variables, if set
 and of length 40.  If all these methods fail, return an anonymous
 authentication.

test/runtests.jl

@@ -451,3 +451,17 @@ end
 finally
     PackageAnalyzer.CATCH_EXCEPTIONS[] = catch_exceptions_value
 end
+
+# This test relies on catching exceptions
+try
+    PackageAnalyzer.CATCH_EXCEPTIONS[] = true
+    @testset "Graceful handling of unparsable files" begin
+        # This codebase includes an uncommented `TODO` which fails to parse
+        # in `sample/test1.jl` we should just skip that file instead of erroing out
+        pkg = analyze("Pluto"; version=v"0.18.0")
+        l_src = PackageAnalyzer.sum_julia_loc(pkg, "src")
+        @test l_src > 6000
+    end
+catch
+    PackageAnalyzer.CATCH_EXCEPTIONS[] = catch_exceptions_value
+end