From 59f937853629d8162f22f6fc5b68c503221fe354 Mon Sep 17 00:00:00 2001 From: Ben Sherman Date: Fri, 18 Jul 2025 17:55:02 -0500 Subject: [PATCH] Type annotations Signed-off-by: Ben Sherman --- docs/migrations/25-10.md | 45 ++++ docs/process.md | 4 +- docs/reference/stdlib-namespaces.md | 5 +- docs/reference/stdlib-types.md | 254 +++++++++++------- docs/script.md | 43 +++ docs/strict-syntax.md | 33 ++- .../script/parser/v2/ScriptCompiler.java | 3 +- .../main/groovy/nextflow/util/ArrayBag.groovy | 2 + modules/nf-commons/build.gradle | 1 + .../src/main/nextflow/util/ArrayTuple.java | 3 +- .../src/main/nextflow/util/Duration.groovy | 8 +- .../src/main/nextflow/util/HashBuilder.java | 1 + .../src/main/nextflow/util/MemoryUnit.groovy | 8 +- .../main/nextflow/util/VersionNumber.groovy | 7 +- .../nf-lang/src/main/antlr/ScriptParser.g4 | 56 +++- .../nextflow/script/ast/ASTNodeMarker.java | 3 + .../java/nextflow/script/ast/OutputNode.java | 5 +- .../java/nextflow/script/ast/ProcessNode.java | 4 +- .../script/ast/ScriptVisitorSupport.java | 1 - .../nextflow/script/ast/WorkflowNode.java | 17 +- .../script/control/ResolveVisitor.java | 56 +++- .../script/control/ScriptResolveVisitor.java | 2 + .../script/control/ScriptToGroovyVisitor.java | 18 +- .../script/control/VariableScopeVisitor.java | 14 +- .../java/nextflow/script/dsl/ScriptDsl.java | 3 +- .../nextflow/script/formatter/Formatter.java | 38 +-- .../formatter/ScriptFormattingVisitor.java | 148 ++++++---- .../script/parser/ScriptAstBuilder.java | 186 ++++++++----- .../main/java/nextflow/script/types/Bag.java | 2 - .../types/{NamedTuple.java => Record.java} | 4 +- .../java/nextflow/script/types/Tuple.java} | 11 +- .../java/nextflow/script/types/Types.java | 8 +- .../nextflow/script/types/shim/String.java | 28 +- .../formatter/ScriptFormatterTest.groovy | 112 +++++++- tests/checks/.IGNORE-PARSER-V2 | 1 + tests/type-annotations.nf | 40 +++ validation/test.sh | 1 + 37 files changed, 842 insertions(+), 333 deletions(-) rename modules/nf-lang/src/main/java/nextflow/script/types/{NamedTuple.java => Record.java} (87%) rename modules/{nf-commons/src/main/nextflow/util/Bag.groovy => nf-lang/src/main/java/nextflow/script/types/Tuple.java} (70%) create mode 100644 tests/type-annotations.nf diff --git a/docs/migrations/25-10.md b/docs/migrations/25-10.md index 97e8c313e8..dd9990e69e 100644 --- a/docs/migrations/25-10.md +++ b/docs/migrations/25-10.md @@ -8,6 +8,51 @@ This page summarizes the upcoming changes in Nextflow 25.10, which will be relea This page is a work in progress and will be updated as features are finalized. It should not be considered complete until the 25.10 release. ::: +## New features + +

Type annotations

+ +Type annotations are a way to denote the *type* of a variable. They are useful both for documenting and validating your pipeline code. + +```nextflow +workflow RNASEQ { + take: + reads: Channel + index: Channel + + main: + samples_ch = QUANT( reads.combine(index) ) + + emit: + samples: Channel = samples_ch +} + +def isSraId(id: String) -> Boolean { + return id.startsWith('SRA') +} +``` + +The following declarations can be annotated with types: + +- Pipeline parameters (the `params` block) +- Workflow takes and emits +- Function parameters and returns +- Local variables +- Closure parameters +- Workflow outputs (the `output` block) + +Type annotations can refer to any of the {ref}`standard types `. + +Type annotations can be appended with `?` to denote that the value can be `null`: + +```nextflow +def x_opt: String? = null +``` + +:::{note} +While Nextflow inherited type annotations of the form ` ` from Groovy, this syntax was deprecated in the {ref}`strict syntax `. Groovy-style type annotations are still allowed for functions and local variables, but will be automatically converted to Nextflow-stype type annotations when formatting code with the language server or `nextflow lint`. +::: + ## Breaking changes - The AWS Java SDK used by Nextflow was upgraded from v1 to v2, which introduced some breaking changes to the `aws.client` config options. See {ref}`the guide ` for details. diff --git a/docs/process.md b/docs/process.md index bde1973d1e..c5fc26d1b9 100644 --- a/docs/process.md +++ b/docs/process.md @@ -655,7 +655,7 @@ hello ### Input tuples (`tuple`) -The `tuple` qualifier allows you to group multiple values into a single input definition. It can be useful when a channel emits tuples of values that need to be handled separately. Each element in the tuple is associated with a corresponding element in the `tuple` definition. For example: +The `tuple` qualifier allows you to group multiple values into a single input definition. It can be useful when a channel emits {ref}`tuples ` of values that need to be handled separately. Each element in the tuple is associated with a corresponding element in the `tuple` definition. For example: ```nextflow process cat { @@ -1056,7 +1056,7 @@ If the command fails, the task will also fail. In Bash, you can append `|| true` ### Output tuples (`tuple`) -The `tuple` qualifier allows you to output multiple values in a single channel. It is useful when you need to associate outputs with metadata, for example: +The `tuple` qualifier allows you to output multiple values in a single channel as a {ref}`tuple `. It is useful when you need to associate outputs with metadata, for example: ```nextflow process blast { diff --git a/docs/reference/stdlib-namespaces.md b/docs/reference/stdlib-namespaces.md index c805b66ad7..c3320ab486 100644 --- a/docs/reference/stdlib-namespaces.md +++ b/docs/reference/stdlib-namespaces.md @@ -105,10 +105,7 @@ The global namespace contains globally available constants and functions. `sleep( milliseconds: long )` : Sleep for the given number of milliseconds. -`tuple( collection: List ) -> ArrayTuple` -: Create a tuple object from the given collection. - -`tuple( args... ) -> ArrayTuple` +`tuple( args... ) -> Tuple` : Create a tuple object from the given arguments. (stdlib-namespaces-channel)= diff --git a/docs/reference/stdlib-types.md b/docs/reference/stdlib-types.md index 9d3edfd99c..d2e28f9acf 100644 --- a/docs/reference/stdlib-types.md +++ b/docs/reference/stdlib-types.md @@ -17,13 +17,29 @@ The following operations are supported for bags: `+ : (Bag, Bag) -> Bag` : Concatenates two bags. -`in, !in : (E, Bag) -> boolean` +`in, !in : (E, Bag) -> Boolean` : Given a value and a bag, returns `true` if the bag contains the value (or not). :::{note} Lists in Nextflow are backed by the [Java](https://docs.oracle.com/en/java/javase/17/docs/api/java.base/java/util/Collection.html) and [Groovy](https://docs.groovy-lang.org/latest/html/groovy-jdk/java/util/Collection.html) standard libraries, which may expose additional methods. Only methods which are recommended for use in Nextflow are documented here. ::: +(stdlib-types-boolean)= + +## Boolean + +A boolean can be `true` or `false`. + +The following operations are supported for booleans: + +- `&&`: logical AND +- `||`: logical OR +- `!`: logical NOT + +:::{note} +Booleans in Nextflow can be backed by any of the following Java types: `boolean`, `Boolean`. +::: + (stdlib-types-channel)= ## Channel\ @@ -36,20 +52,20 @@ See {ref}`channel-page` for an overview of channels. See {ref}`channel-factory` A Duration represents a duration of time with millisecond precision. -A Duration can be created by adding a unit suffix to an integer (e.g. `1.h`), or more explicitly with `Duration.of()`: +A Duration can be created by adding a unit suffix to an integer (e.g. `1.h`), or by explicitly casting to `Duration`: ```nextflow // integer with suffix oneDay = 24.h // integer value (milliseconds) -oneSecond = Duration.of(1000) +oneSecond = 1000 as Duration // simple string value -oneHour = Duration.of('1h') +oneHour = '1h' as Duration // complex string value -complexDuration = Duration.of('1day 6hours 3minutes 30seconds') +complexDuration = '1day 6hours 3minutes 30seconds' as Duration ``` The following suffixes are available: @@ -77,38 +93,71 @@ assert b / 2 == a The following methods are available for a Duration: -`toDays() -> long` +`toDays() -> Integer` : Get the duration value in days (rounded down). -`toHours() -> long` +`toHours() -> Integer` : Get the duration value in hours (rounded down). -`toMillis() -> long` +`toMillis() -> Integer` : Get the duration value in milliseconds. -`toMinutes() -> long` +`toMinutes() -> Integer` : Get the duration value in minutes (rounded down). -`toSeconds() -> long` +`toSeconds() -> Integer` : Get the duration value in seconds (rounded down). :::{note} These methods are also available as `getDays()`, `getHours()`, `getMillis()`, `getMinutes()`, and `getSeconds()`. ::: +(stdlib-types-float)= + +## Float + +A float is a floating-point number (i.e. real number) that can be positive or negative. + +The following operations are supported for floats: + +- `+, -, *, /`: addition, subtraction, multiplication, division +- `**`: exponentation + +:::{note} +Floats in Nextflow can be backed by any of the following Java types: `float`, `double`, `Float`, `Double`. +::: + +(stdlib-types-integer)= + +## Integer + +An integer is a whole number that can be positive or negative. + +The following operations are supported for integers: + +- `+, -, *, /`: addition, subtraction, multiplication, integer division +- `%`: remainder of integer division (i.e. modulus) +- `**`: exponentation +- `&`, `|`, `^`, `~`: bitwise AND, OR, XOR, NOT +- `<<`, `>>`: left and right bit-shift + +:::{note} +Integers in Nextflow can be backed by any of the following Java types: `int`, `long`, `Integer`, `Long`. +::: + (stdlib-types-iterable)= ## Iterable\ *Implemented by the following types: {ref}`stdlib-types-bag`, {ref}`stdlib-types-list`, {ref}`stdlib-types-set`* -An iterable is a trait implemented by collection types that support iteration. +`Iterable` is a trait implemented by collection types that support iteration. Types that implement `Iterable` can be passed as an `Iterable` parameter of a method, and they can use all of the methods described below. The following methods are available for iterables: -`any( condition: (E) -> boolean ) -> boolean` +`any( condition: (E) -> Boolean ) -> Boolean` : Returns `true` if any element in the iterable satisfies the given condition. `collect( transform: (E) -> R ) -> Iterable` @@ -117,16 +166,16 @@ The following methods are available for iterables: `collectMany( transform: (E) -> Iterable ) -> Iterable` : Transforms each element in the iterable into a collection with the given closure and concatenates the resulting collections into a list. -`contains( value: E ) -> boolean` +`contains( value: E ) -> Boolean` : Returns `true` if the iterable contains the given value. `each( action: (E) -> () )` : Invoke the given closure for each element in the iterable. -`every( condition: (E) -> boolean ) -> boolean` +`every( condition: (E) -> Boolean ) -> Boolean` : Returns `true` if every element in the iterable satisfies the given condition. -`findAll( condition: (E) -> boolean ) -> Iterable` +`findAll( condition: (E) -> Boolean ) -> Iterable` : Returns the elements in the iterable that satisfy the given condition. `groupBy( transform: (E) -> K ) -> Map>` @@ -139,7 +188,7 @@ The following methods are available for iterables: `inject( initialValue: R, accumulator: (R,E) -> R ) -> R` : Apply the given accumulator to each element in the iterable and return the final accumulated value. The closure should accept two parameters, corresponding to the current accumulated value and the current iterable element, and return the next accumulated value. -`isEmpty() -> boolean` +`isEmpty() -> Boolean` : Returns `true` if the iterable is empty. `join( separator: String = '' ) -> String` @@ -150,7 +199,7 @@ The following methods are available for iterables: **`max( comparator: (E) -> R ) -> E`** -`max( comparator: (E,E) -> int ) -> E` +`max( comparator: (E,E) -> Integer ) -> E` : Returns the maximum element in the iterable according to the given closure. : The closure should follow the same semantics as the closure parameter of `toSorted()`. @@ -159,11 +208,11 @@ The following methods are available for iterables: **`min( comparator: (E) -> R ) -> E`** -`min( comparator: (E,E) -> int ) -> E` +`min( comparator: (E,E) -> Integer ) -> E` : Returns the maximum element in the iterable according to the given closure. : The closure should follow the same semantics as the closure parameter of `toSorted()`. -`size() -> int` +`size() -> Integer` : Returns the number of elements in the iterable. `sum() -> E` @@ -188,7 +237,7 @@ The following methods are available for iterables: : Returns the iterable as a list sorted according to the given closure. : The closure should accept one parameter and transform each element into the value that will be used for comparisons. -`toSorted( comparator: (E,E) -> int ) -> List` +`toSorted( comparator: (E,E) -> Integer ) -> List` : Returns the iterable as a list sorted according to the given closure. : The closure should accept two parameters and return a negative integer, zero, or a positive integer to denote whether the first argument is less than, equal to, or greater than the second. @@ -197,7 +246,7 @@ The following methods are available for iterables: **`toUnique( comparator: (E) -> R ) -> Iterable`** -`toUnique( comparator: (E,E) -> int ) -> Iterable` +`toUnique( comparator: (E,E) -> Integer ) -> Iterable` : Returns a shallow copy of the iterable with duplicate elements excluded. : The closure should follow the same semantics as the closure parameter of `toSorted()`. @@ -218,18 +267,18 @@ The following operations are supported for lists: `+ : (List, List) -> List` : Concatenates two lists. -`* : (List, int) -> List` +`* : (List, Integer) -> List` : Given a list and an integer *n*, repeats the list *n* times. -`[] : (List, int) -> E` +`[] : (List, Integer) -> E` : Given a list and an index, returns the element at the given index in the list, or `null` if the index is out of range. -`in, !in : (E, List) -> boolean` +`in, !in : (E, List) -> Boolean` : Given a value and a list, returns `true` if the list contains the value (or not). The following methods are available for a list: -`collate( size: int, keepRemainder: boolean = true ) -> List>` +`collate( size: Integer, keepRemainder: Boolean = true ) -> List>` : Collates the list into a list of sub-lists of length `size`. If `keepRemainder` is `true`, any remaining elements are included as a partial sub-list, otherwise they are excluded. : For example: @@ -238,7 +287,7 @@ The following methods are available for a list: assert [1, 2, 3, 4, 5, 6, 7].collate(3, false) == [[1, 2, 3], [4, 5, 6]] ``` -`collate( size: int, step: int, keepRemainder: boolean = true ) -> List>` +`collate( size: Integer, step: Integer, keepRemainder: Boolean = true ) -> List>` : Collates the list into a list of sub-lists of length `size`, stepping through the list `step` elements for each sub-list. If `keepRemainder` is `true`, any remaining elements are included as a partial sub-list, otherwise they are excluded. : For example: @@ -247,7 +296,7 @@ The following methods are available for a list: assert [1, 2, 3, 4].collate(3, 1, false) == [[1, 2, 3], [2, 3, 4]] ``` -`find( condition: (E) -> boolean ) -> E` +`find( condition: (E) -> Boolean ) -> E` : Returns the first element in the list that satisfies the given condition. `first() -> E` @@ -259,7 +308,7 @@ The following methods are available for a list: `head() -> E` : Equivalent to `first()`. -`indexOf( value: E ) -> int` +`indexOf( value: E ) -> Integer` : Returns the index of the first occurrence of the given value in the list, or -1 if the list does not contain the value. `init() -> List` @@ -271,16 +320,16 @@ The following methods are available for a list: `reverse() -> List` : Returns a shallow copy of the list with the elements reversed. -`subList( fromIndex: int, toIndex: int ) -> List` +`subList( fromIndex: Integer, toIndex: Integer ) -> List` : Returns the portion of the list between the given `fromIndex` (inclusive) and `toIndex` (exclusive). `tail() -> List` : Returns a shallow copy of the list with the first element excluded. -`take( n: int ) -> List` +`take( n: Integer ) -> List` : Returns the first *n* elements of the list. -`takeWhile( condition: (E) -> boolean ) -> List` +`takeWhile( condition: (E) -> Boolean ) -> List` : Returns the longest prefix of the list where each element satisfies the given condition. `withIndex() -> List<(E,Integer)>` @@ -304,18 +353,18 @@ The following operations are supported for maps: `[] : (Map, K) -> V` : Given a map and a key, returns the value for the given key in the map, or `null` if the key is not in the map. -`in, !in : (K, Map) -> boolean` +`in, !in : (K, Map) -> Boolean` : Given a key and a map, returns `true` if the map contains the key and the corresponding value is *truthy* (e.g. not `null`, `0`, or `false`). The following methods are available for a map: -`any( condition: (K,V) -> boolean ) -> boolean` +`any( condition: (K,V) -> Boolean ) -> Boolean` : Returns `true` if any key-value pair in the map satisfies the given condition. The closure should accept two parameters corresponding to the key and value of an entry. -`containsKey( key: K ) -> boolean` +`containsKey( key: K ) -> Boolean` : Returns `true` if the map contains a mapping for the given key. -`containsValue( value: V ) -> boolean` +`containsValue( value: V ) -> Boolean` : Returns `true` if the map maps one or more keys to the given value. `each( action: (K,V) -> () )` @@ -324,16 +373,16 @@ The following methods are available for a map: `entrySet() -> Set<(K,V)>` : Returns a set of the key-value pairs in the map. -`every( condition: (K,V) -> boolean ) -> boolean` +`every( condition: (K,V) -> Boolean ) -> Boolean` : Returns `true` if every key-value pair in the map satisfies the given condition. The closure should accept two parameters corresponding to the key and value of an entry. -`isEmpty() -> boolean` +`isEmpty() -> Boolean` : Returns `true` if the map is empty. `keySet() -> Set` : Returns a set of the keys in the map. -`size() -> int` +`size() -> Integer` : Returns the number of key-value pairs in the map. `subMap( keys: Iterable ) -> Map` @@ -352,17 +401,17 @@ Maps in Nextflow are backed by the [Java](https://docs.oracle.com/en/java/javase A MemoryUnit represents a quantity of bytes. -A MemoryUnit can be created by adding a unit suffix to an integer (e.g. `1.GB`), or more explicitly with `MemoryUnit.of()`: +A MemoryUnit can be created by adding a unit suffix to an integer (e.g. `1.GB`), or by explicitly casting to `MemoryUnit`: ```nextflow // integer with suffix oneMegabyte = 1.MB // integer value (bytes) -oneKilobyte = MemoryUnit.of(1024) +oneKilobyte = 1024 as MemoryUnit // string value -oneGigabyte = MemoryUnit.of('1 GB') +oneGigabyte = '1 GB' as MemoryUnit ``` The following suffixes are available: @@ -397,19 +446,19 @@ assert b / 2 == a The following methods are available for a `MemoryUnit` object: -`toBytes() -> long` +`toBytes() -> Integer` : Get the memory value in bytes (B). -`toGiga() -> long` +`toGiga() -> Integer` : Get the memory value in gigabytes (rounded down), where 1 GB = 1024 MB. -`toKilo() -> long` +`toKilo() -> Integer` : Get the memory value in kilobytes (rounded down), where 1 KB = 1024 B. -`toMega() -> long` +`toMega() -> Integer` : Get the memory value in megabytes (rounded down), where 1 MB = 1024 KB. -`toUnit( unit: String ) -> long` +`toUnit( unit: String ) -> Integer` : Get the memory value in terms of a given unit (rounded down). The unit can be one of: `'B'`, `'KB'`, `'MB'`, `'GB'`, `'TB'`, `'PB'`, `'EB'`, `'ZB'`. :::{note} @@ -458,7 +507,7 @@ The following operations are supported for paths: The following methods are useful for getting attributes of a path: -`exists() -> boolean` +`exists() -> Boolean` : Returns `true` if the path exists. `getBaseName() -> String` @@ -479,22 +528,22 @@ The following methods are useful for getting attributes of a path: `getScheme() -> String` : Gets the path URI scheme, e.g. `s3://some-bucket/hello.txt` -> `s3`. -`isDirectory() -> boolean` +`isDirectory() -> Boolean` : Returns `true` if the path is a directory. -`isEmpty() -> boolean` +`isEmpty() -> Boolean` : Returns `true` if the path is empty or does not exist. -`isFile() -> boolean` +`isFile() -> Boolean` : Returns `true` if the path is a file (i.e. not a directory). -`isHidden() -> boolean` +`isHidden() -> Boolean` : Returns `true` if the path is hidden. -`isLink() -> boolean` +`isLink() -> Boolean` : Returns `true` if the path is a symbolic link. -`lastModified() -> long` +`lastModified() -> Integer` : Returns the path last modified timestamp in Unix time (i.e. milliseconds since January 1, 1970). `relativize(other: Path) -> Path` @@ -506,7 +555,7 @@ The following methods are useful for getting attributes of a path: `resolveSibling(other: String) -> Path` : Resolves the given path string against this path's parent path. -`size() -> long` +`size() -> Integer` : Gets the file size in bytes. `toUriString() -> String` @@ -615,7 +664,7 @@ The following methods are available for manipulating files and directories in a The `copyTo()` function follows the semantics of the Linux command `cp -r `, with the following caveat: while Linux tools often treat paths ending with a slash (e.g. `/some/path/name/`) as directories, and those not (e.g. `/some/path/name`) as regular files, Nextflow (due to its use of the Java files API) views both of these paths as the same file system object. If the path exists, it is handled according to its actual type (i.e. as a regular file or as a directory). If the path does not exist, it is treated as a regular file, with any missing parent directories created automatically. ::: -`delete() -> boolean` +`delete() -> Boolean` : Deletes the file or directory at the given path, returning `true` if the operation succeeds, and `false` otherwise: ```nextflow @@ -626,7 +675,7 @@ The following methods are available for manipulating files and directories in a If a directory is not empty, it will not be deleted and `delete()` will return `false`. -`deleteDir() -> boolean` +`deleteDir() -> Boolean` : Deletes a directory and all of its contents. ```nextflow @@ -642,7 +691,7 @@ The following methods are available for manipulating files and directories in a `listFiles() -> List` : Returns the first-level elements (files and directories) of a directory as a list of Paths. -`mkdir() -> boolean` +`mkdir() -> Boolean` : Creates a directory at the given path, returning `true` if the directory is created successfully, and `false` otherwise: ```nextflow @@ -653,7 +702,7 @@ The following methods are available for manipulating files and directories in a If the parent directories do not exist, the directory will not be created and `mkdir()` will return `false`. -`mkdirs() -> boolean` +`mkdirs() -> Boolean` : Creates a directory at the given path, including any nonexistent parent directories: ```nextflow @@ -672,30 +721,30 @@ The following methods are available for manipulating files and directories in a Available options: - `hard: boolean` + `hard: Boolean` : When `true`, creates a *hard* link, otherwise creates a *soft* (aka *symbolic*) link (default: `false`). - `overwrite: boolean` + `overwrite: Boolean` : When `true`, overwrites any existing file with the same name, otherwise throws a [FileAlreadyExistsException](https://docs.oracle.com/en/java/javase/17/docs/api/java.base/java/nio/file/FileAlreadyExistsException.html) (default: `false`). `moveTo( target: Path )` : Moves a source file or directory to a target file or directory. Follows the same semantics as `copyTo()`. -`renameTo( target: String ) -> boolean` +`renameTo( target: String ) -> Boolean` : Rename a file or directory: ```nextflow file('my_file.txt').renameTo('new_file_name.txt') ``` -`setPermissions( permissions: String ) -> boolean` +`setPermissions( permissions: String ) -> Boolean` : Sets a file's permissions using the [symbolic notation](http://en.wikipedia.org/wiki/File_system_permissions#Symbolic_notation): ```nextflow myFile.setPermissions('rwxr-xr-x') ``` -`setPermissions( owner: int, group: int, other: int ) -> boolean` +`setPermissions( owner: Integer, group: Integer, other: Integer ) -> Boolean` : Sets a file's permissions using the [numeric notation](http://en.wikipedia.org/wiki/File_system_permissions#Numeric_notation), i.e. as three digits representing the **owner**, **group**, and **other** permissions: ```nextflow @@ -726,16 +775,16 @@ The following methods are available for listing and traversing directories: The following methods are available for splitting and counting the records in files: -`countFasta() -> long` +`countFasta() -> Integer` : Counts the number of records in a [FASTA](https://en.wikipedia.org/wiki/FASTA_format) file. See the {ref}`operator-splitfasta` operator for available options. -`countFastq() -> long` +`countFastq() -> Integer` : Counts the number of records in a [FASTQ](https://en.wikipedia.org/wiki/FASTQ_format) file. See the {ref}`operator-splitfastq` operator for available options. -`countJson() -> long` +`countJson() -> Integer` : Counts the number of records in a JSON file. See the {ref}`operator-splitjson` operator for available options. -`countLines() -> long` +`countLines() -> Integer` : Counts the number of lines in a text file. See the {ref}`operator-splittext` operator for available options. `splitCsv() -> List` @@ -761,7 +810,7 @@ The following methods are available for splitting and counting the records in fi A set is an unordered collection that cannot contain duplicate elements. -As set literal can be created from a list: +A set can be created from a list using the `toSet()` method: ```nextflow [1, 2, 2, 3].toSet() @@ -776,7 +825,7 @@ The following operations are supported for sets: `- : (Set, Iterable) -> Set` : Given a set and an iterable, returns a shallow copy of the set minus the elements of the iterable. -`in, !in : (E, Set) -> boolean` +`in, !in : (E, Set) -> Boolean` : Given a value and a set, returns `true` if the set contains the value (or not). The following methods are available for a set: @@ -799,13 +848,13 @@ The following operations are supported for strings: `+ : (String, String) -> String` : Concatenates two strings. -`* : (String, int) -> String` +`* : (String, Integer) -> String` : Given a string and an integer *n*, repeats the string *n* times. -`[] : (String, int) -> char` +`[] : (String, Integer) -> char` : Given a string and an index, returns the character at the given index in the string. -`in, !in : (String, String) -> boolean` +`in, !in : (String, String) -> Boolean` : Given a substring and a string, returns `true` if the substring occurs anywhere in the string (or not). `~ : (String) -> Pattern` @@ -816,42 +865,48 @@ The following operations are supported for strings: : Given a string and a pattern, creates a matcher that is truthy if the pattern occurs anywhere in the string. : See [Matcher](https://docs.oracle.com/en/java/javase/17/docs/api/java.base/java/util/regex/Matcher.html) in the Java standard library for more information. -`==~ : (String, String) -> boolean` +`==~ : (String, String) -> Boolean` : Given a string and a pattern, returns `true` if the string matches the pattern exactly. The following methods are available for a string: -`endsWith( suffix: String ) -> boolean` +`endsWith( suffix: String ) -> Boolean` : Returns `true` if the string ends with the given suffix. `execute() -> Process` : Execute the string as a command. Returns a [Process](https://docs.groovy-lang.org/latest/html/groovy-jdk/java/lang/Process.html) which provides the exit status and standard input/output/error of the executed command. -`indexOf( str: String ) -> int` +`indexOf( str: String ) -> Integer` : Returns the index within the string of the first occurrence of the given substring. Returns -1 if the string does not contain the substring. -`indexOf( str: String, fromIndex: int ) -> int` +`indexOf( str: String, fromIndex: Integer ) -> Integer` : Returns the index within the string of the first occurrence of the given substring, starting the search at the given index. Returns -1 if the string does not contain the substring. -`isBlank() -> boolean` +`isBlank() -> Boolean` : Returns `true` if the string is empty or contains only whitespace characters. -`isEmpty() -> boolean` +`isEmpty() -> Boolean` : Returns `true` if the string is empty (i.e. `length()` is 0). -`isFloat() -> boolean` -: Returns `true` if the string can be parsed as a floating-point number. +`isDouble() -> Boolean` +: Returns `true` if the string can be parsed as a 64-bit floating-point number. + +`isFloat() -> Boolean` +: Returns `true` if the string can be parsed as a 32-bit floating-point number. -`isInteger() -> boolean` -: Returns `true` if the string can be parsed as an integer. +`isInteger() -> Boolean` +: Returns `true` if the string can be parsed as a 32-bit integer. -`lastIndexOf( str: String ) -> int` +`isLong() -> Boolean` +: Returns `true` if the string can be parsed as a 64-bit integer. + +`lastIndexOf( str: String ) -> Integer` : Returns the index within the string of the last occurrence of the given substring. Returns -1 if the string does not contain the substring. -`lastIndexOf( str: String, fromIndex: int ) -> int` +`lastIndexOf( str: String, fromIndex: Integer ) -> Integer` : Returns the index within the string of the last occurrence of the given substring, searching backwards starting at the given index. Returns -1 if the string does not contain the substring. -`length() -> int` +`length() -> Integer` : Returns the length of the string. `md5() -> String` @@ -869,7 +924,7 @@ The following methods are available for a string: `sha256() -> String` : Returns the SHA-256 checksum of the string. -`startsWith( prefix: String ) -> boolean` +`startsWith( prefix: String ) -> Boolean` : Returns `true` if the string ends with the given prefix. `strip() -> String` @@ -885,20 +940,26 @@ The following methods are available for a string: `stripTrailing() -> String` : Returns a copy of the string with all trailing whitespace removed. -`substring( beginIndex: int ) -> String` +`substring( beginIndex: Integer ) -> String` : Returns a substring of this string. -`substring( beginIndex: int, endIndex: int ) -> String` +`substring( beginIndex: Integer, endIndex: Integer ) -> String` : Returns a substring of this string. `toBoolean() -> Boolean` : Returns `true` if the trimmed string is "true", "y", or "1" (ignoring case). +`toDouble() -> Float` +: Parses the string into a 64-bit floating-point number. + `toFloat() -> Float` -: Parses the string into a floating-point number. +: Parses the string into a 32-bit floating-point number. `toInteger() -> Integer` -: Parses the string into an integer. +: Parses the string into a 32-bit integer. + +`toLong() -> Integer` +: Parses the string into a 64-bit integer. `toLowerCase() -> String` : Returns a copy of this string with all characters converted to lower case. @@ -913,6 +974,17 @@ The following methods are available for a string: Strings in Nextflow are backed by the [Java](https://docs.oracle.com/en/java/javase/17/docs/api/java.base/java/lang/String.html) and [Groovy](https://docs.groovy-lang.org/latest/html/groovy-jdk/java/lang/String.html) standard libraries, which may expose additional methods. Only methods which are recommended for use in Nextflow are documented here. ::: +(stdlib-types-tuple)= + +## Tuple + +A tuple is a fixed sequence of values, where each value can have its own type. Tuples can be created using the `tuple` function. + +The following operations are supported for tuples: + +`[] : (Tuple, Integer) -> ?` +: Given a tuple and an index, returns the tuple element at the index. + (stdlib-types-versionnumber)= ## VersionNumber @@ -930,7 +1002,7 @@ The following methods are available for a VersionNumber: `getPatch() -> String` : Get the patch version number, i.e. the third version component. -`matches( condition: String ) -> boolean` +`matches( condition: String ) -> Boolean` : Check whether the version satisfies a version requirement. : The version requirement string can be prefixed with the usual comparison operators: diff --git a/docs/script.md b/docs/script.md index 578f56b50a..957b116693 100644 --- a/docs/script.md +++ b/docs/script.md @@ -111,6 +111,49 @@ Copying a map with the `+` operator is a safer way to modify maps in Nextflow, s See {ref}`stdlib-types-map` for the set of available map operations. +(script-tuples)= + +## Tuples + +Tuples are used to store a fixed sequence of heterogeneous values. They are created using the `tuple` function: + +```nextflow +person = tuple('Alice', 42, false) +``` + +Tuple elements are accessed by index: + +```nextflow +name = person[0] +age = person[1] +is_male = person[2] +``` + +Tuples can be destructured in assignments: + +```nextflow +(name, age, is_male) = person +``` + +As well as closure parameters: + +```nextflow +coords = [ + tuple(1, 2), + tuple(2, 4), + tuple(3, 6), + tuple(4, 8) +] + +coords.each { x, y -> + println "x=$x, y=$y" +} +``` + +Tuples are immutable -- once a tuple is created, its elements cannot be modified. + +See {ref}`stdlib-types-tuple` for the set of available tuple operations. + (script-operators)= ## Operators diff --git a/docs/strict-syntax.md b/docs/strict-syntax.md index d5cac9a7b2..dd4c30f187 100644 --- a/docs/strict-syntax.md +++ b/docs/strict-syntax.md @@ -299,10 +299,22 @@ def str = 'hello' def meta = [:] ``` -:::{note} -Because type annotations are useful for providing type checking at runtime, the language server will not report errors for Groovy-style type annotations at this time. Type annotations will be addressed in a future version of the Nextflow language specification. +:::{versionadded} 25.10.0 ::: +Local variables can be declared with a type annotation: + +```nextflow +def a: Integer = 1 +def b: Integer = 2 +def (c: Integer, d: Integer) = [3, 4] +def (e: Integer, f: Integer) = [5, 6] +def str: String = 'hello' +def meta: Map = [:] +``` + +While Groovy-style type annotations are still supported, the linter and language server will automatically convert them to Nextflow-style type annotations when formatting code. Groovy-style type annotations will not be supported in a future version. + ### Strings Groovy supports a wide variety of strings, including multi-line strings, dynamic strings, slashy strings, multi-line dynamic slashy strings, and more. @@ -368,21 +380,26 @@ def map = (Map) readJson(json) // soft cast def map = readJson(json) as Map // hard cast ``` -In the strict syntax, only hard casts are supported. However, hard casts are discouraged because they can cause unexpected behavior if used improperly. Groovy-style type annotations should be used instead: +In the strict syntax, only hard casts are supported. + +When casting a value to a different type, it is always better to use an explicit method if one is available. For example, to parse a string as a number: ```groovy -def Map map = readJson(json) +def x = '42' as Integer +def x = '42'.toInteger() // preferred ``` -Nextflow will raise an error at runtime if the `readJson()` function does not return a `Map`. +:::{versionadded} 25.10.0 +::: -When converting a value to a different type, it is better to use an explicit method rather than a cast. For example, to parse a string as a number: +In cases where a function returns an unknown type, use a type annotation: ```groovy -def x = '42' as Integer -def x = '42'.toInteger() // preferred +def map: Map = readJson(json) ``` +Nextflow will raise an error at runtime if the `readJson()` function does not return a `Map`. + ### Process env inputs and outputs In Nextflow DSL2, the name of a process `env` input/output can be specified with or without quotes: diff --git a/modules/nextflow/src/main/groovy/nextflow/script/parser/v2/ScriptCompiler.java b/modules/nextflow/src/main/groovy/nextflow/script/parser/v2/ScriptCompiler.java index dab1c55644..32a85164d8 100644 --- a/modules/nextflow/src/main/groovy/nextflow/script/parser/v2/ScriptCompiler.java +++ b/modules/nextflow/src/main/groovy/nextflow/script/parser/v2/ScriptCompiler.java @@ -68,7 +68,8 @@ public class ScriptCompiler { "java.nio.file.Path", "nextflow.Channel", "nextflow.util.Duration", - "nextflow.util.MemoryUnit" + "nextflow.util.MemoryUnit", + "nextflow.util.VersionNumber" ); private static final String MAIN_CLASS_NAME = "Main"; private static final String BASE_CLASS_NAME = "nextflow.script.BaseScript"; diff --git a/modules/nextflow/src/main/groovy/nextflow/util/ArrayBag.groovy b/modules/nextflow/src/main/groovy/nextflow/util/ArrayBag.groovy index bd7dcff3b8..947fb5b6ce 100644 --- a/modules/nextflow/src/main/groovy/nextflow/util/ArrayBag.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/util/ArrayBag.groovy @@ -15,11 +15,13 @@ */ package nextflow.util + import com.esotericsoftware.kryo.Kryo import com.esotericsoftware.kryo.KryoSerializable import com.esotericsoftware.kryo.io.Input import com.esotericsoftware.kryo.io.Output import groovy.transform.CompileStatic +import nextflow.script.types.Bag import org.codehaus.groovy.runtime.InvokerHelper /** diff --git a/modules/nf-commons/build.gradle b/modules/nf-commons/build.gradle index f687d89fae..02da8208c1 100644 --- a/modules/nf-commons/build.gradle +++ b/modules/nf-commons/build.gradle @@ -25,6 +25,7 @@ sourceSets { } dependencies { + api(project(':nf-lang')) api "ch.qos.logback:logback-classic:1.5.18" api "org.apache.groovy:groovy:4.0.27" api "org.apache.groovy:groovy-nio:4.0.27" diff --git a/modules/nf-commons/src/main/nextflow/util/ArrayTuple.java b/modules/nf-commons/src/main/nextflow/util/ArrayTuple.java index feb3034cca..6650e97e2b 100644 --- a/modules/nf-commons/src/main/nextflow/util/ArrayTuple.java +++ b/modules/nf-commons/src/main/nextflow/util/ArrayTuple.java @@ -21,6 +21,7 @@ import java.util.Iterator; import java.util.List; import java.util.ListIterator; +import nextflow.script.types.Tuple; /** * Provides a basic tuple implementation extending an {@link ArrayList} @@ -28,7 +29,7 @@ * * @author Paolo Di Tommaso */ -public class ArrayTuple extends ArrayList { +public class ArrayTuple extends ArrayList implements Tuple { private static final long serialVersionUID = - 4765828600345948947L; diff --git a/modules/nf-commons/src/main/nextflow/util/Duration.groovy b/modules/nf-commons/src/main/nextflow/util/Duration.groovy index 54d0239059..a5e1636e6b 100644 --- a/modules/nf-commons/src/main/nextflow/util/Duration.groovy +++ b/modules/nf-commons/src/main/nextflow/util/Duration.groovy @@ -22,6 +22,7 @@ import java.util.concurrent.TimeUnit import groovy.transform.CompileStatic import groovy.transform.EqualsAndHashCode import groovy.util.logging.Slf4j +import nextflow.script.types.Duration as IDuration import org.apache.commons.lang.time.DurationFormatUtils /** * A simple time duration representation @@ -31,7 +32,7 @@ import org.apache.commons.lang.time.DurationFormatUtils @Slf4j @CompileStatic @EqualsAndHashCode(includes = 'durationInMillis') -class Duration implements Comparable, Serializable, Cloneable { +class Duration implements IDuration, Comparable, Serializable, Cloneable { static private final FORMAT = ~/^(\d+\.?\d*)\s*([a-zA-Z]+)/ @@ -215,6 +216,7 @@ class Duration implements Comparable, Serializable, Cloneable { new Duration(java.time.Duration.between(start, end).toMillis()) } + @Override long toMillis() { durationInMillis } @@ -223,6 +225,7 @@ class Duration implements Comparable, Serializable, Cloneable { durationInMillis } + @Override long toSeconds() { TimeUnit.MILLISECONDS.toSeconds(durationInMillis) } @@ -231,6 +234,7 @@ class Duration implements Comparable, Serializable, Cloneable { toSeconds() } + @Override long toMinutes() { TimeUnit.MILLISECONDS.toMinutes(durationInMillis) } @@ -239,6 +243,7 @@ class Duration implements Comparable, Serializable, Cloneable { toMinutes() } + @Override long toHours() { TimeUnit.MILLISECONDS.toHours(durationInMillis) } @@ -247,6 +252,7 @@ class Duration implements Comparable, Serializable, Cloneable { toHours() } + @Override long toDays() { TimeUnit.MILLISECONDS.toDays(durationInMillis) } diff --git a/modules/nf-commons/src/main/nextflow/util/HashBuilder.java b/modules/nf-commons/src/main/nextflow/util/HashBuilder.java index 6ac234d57a..c9ea56777e 100644 --- a/modules/nf-commons/src/main/nextflow/util/HashBuilder.java +++ b/modules/nf-commons/src/main/nextflow/util/HashBuilder.java @@ -46,6 +46,7 @@ import nextflow.extension.FilesEx; import nextflow.file.FileHolder; import nextflow.io.SerializableMarker; +import nextflow.script.types.Bag; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/modules/nf-commons/src/main/nextflow/util/MemoryUnit.groovy b/modules/nf-commons/src/main/nextflow/util/MemoryUnit.groovy index 26495a81f4..9461f2a887 100644 --- a/modules/nf-commons/src/main/nextflow/util/MemoryUnit.groovy +++ b/modules/nf-commons/src/main/nextflow/util/MemoryUnit.groovy @@ -22,6 +22,7 @@ import java.util.regex.Pattern import groovy.transform.CompileStatic import groovy.transform.EqualsAndHashCode +import nextflow.script.types.MemoryUnit as IMemoryUnit /** * Represent a memory unit * @@ -29,7 +30,7 @@ import groovy.transform.EqualsAndHashCode */ @CompileStatic @EqualsAndHashCode(includes = 'size', includeFields = true) -class MemoryUnit implements Comparable, Serializable, Cloneable { +class MemoryUnit implements IMemoryUnit, Comparable, Serializable, Cloneable { final static public MemoryUnit ZERO = new MemoryUnit(0) @@ -95,20 +96,24 @@ class MemoryUnit implements Comparable, Serializable, Cloneable { } + @Override long toBytes() { size } long getBytes() { size } + @Override long toKilo() { size >> 10 } long getKilo() { size >> 10 } + @Override long toMega() { size >> 20 } long getMega() { size >> 20 } + @Override long toGiga() { size >> 30 } long getGiga() { size >> 30 } @@ -181,6 +186,7 @@ class MemoryUnit implements Comparable, Serializable, Cloneable { * * @param unit String expressing memory unit in bytes, e.g. KB, MB, GB */ + @Override long toUnit(String unit){ int p = UNITS.indexOf(unit) if (p==-1) diff --git a/modules/nf-commons/src/main/nextflow/util/VersionNumber.groovy b/modules/nf-commons/src/main/nextflow/util/VersionNumber.groovy index 9e74ccaec4..bb359c797a 100644 --- a/modules/nf-commons/src/main/nextflow/util/VersionNumber.groovy +++ b/modules/nf-commons/src/main/nextflow/util/VersionNumber.groovy @@ -20,6 +20,7 @@ import java.util.regex.Pattern import groovy.transform.CompileStatic import nextflow.extension.Bolts +import nextflow.script.types.VersionNumber as IVersionNumber /** * Model a semantic version number @@ -27,7 +28,7 @@ import nextflow.extension.Bolts * @author Paolo Di Tommaso */ @CompileStatic -class VersionNumber implements Comparable { +class VersionNumber implements IVersionNumber, Comparable { static private Pattern CHECK = ~/\s*([!<=>]+)?\s*([0-9A-Za-z\-_\.]+)(\+)?/ @@ -62,16 +63,19 @@ class VersionNumber implements Comparable { /** * @return The major version number ie. the first component */ + @Override String getMajor() { version[0] } /** * @return The minor version number ie. the second component */ + @Override String getMinor() { version[1] } /** * @return The minor version number ie. the third component */ + @Override String getPatch() { version[2] } /** @@ -177,6 +181,7 @@ class VersionNumber implements Comparable { * @param condition * @return */ + @Override boolean matches(String condition) { for( String token : condition.tokenize(',')) { if( !matches0(token) ) diff --git a/modules/nf-lang/src/main/antlr/ScriptParser.g4 b/modules/nf-lang/src/main/antlr/ScriptParser.g4 index f35740d7b9..10e76f09bb 100644 --- a/modules/nf-lang/src/main/antlr/ScriptParser.g4 +++ b/modules/nf-lang/src/main/antlr/ScriptParser.g4 @@ -243,7 +243,12 @@ workflowBody ; workflowTakes - : identifier (sep identifier)* + : workflowTake (sep workflowTake)* + ; + +workflowTake + : identifier (COLON type)? + | statement ; workflowMain @@ -251,11 +256,16 @@ workflowMain ; workflowEmits - : statement (sep statement)* + : workflowEmit (sep workflowEmit)* + ; + +workflowEmit + : nameTypePair (ASSIGN expression)? + | statement ; workflowPublishers - : statement (sep statement)* + : workflowEmit (sep workflowEmit)* ; // -- output definition @@ -270,14 +280,19 @@ outputBody ; outputDeclaration - : identifier LBRACE nls blockStatements? RBRACE + : identifier (COLON type)? LBRACE nls blockStatements? RBRACE | statement ; // -- function definition functionDef - : (DEF | legacyType | DEF legacyType) identifier LPAREN nls (formalParameterList nls)? rparen nls LBRACE - nls blockStatements? RBRACE + : DEF + identifier LPAREN nls (formalParameterList nls)? rparen (ARROW type)? + nls LBRACE nls blockStatements? RBRACE + + | (legacyType | DEF legacyType) + identifier LPAREN nls (formalParameterList nls)? rparen + nls LBRACE nls blockStatements? RBRACE ; // -- incomplete script declaration @@ -322,7 +337,12 @@ tryCatchStatement ; catchClause - : CATCH LPAREN catchTypes? identifier rparen nls statementOrBlock + : CATCH LPAREN catchVariable rparen nls statementOrBlock + ; + +catchVariable + : identifier (COLON catchTypes)? + | catchTypes identifier ; catchTypes @@ -336,12 +356,17 @@ assertStatement // -- variable declaration variableDeclaration - : (DEF | legacyType | DEF legacyType) identifier (nls ASSIGN nls initializer=expression)? - | DEF variableNames nls ASSIGN nls initializer=expression + : DEF nameTypePair (nls ASSIGN nls initializer=expression)? + | DEF nameTypePairs nls ASSIGN nls initializer=expression + | (legacyType | DEF legacyType) identifier (nls ASSIGN nls initializer=expression)? ; -variableNames - : LPAREN identifier (COMMA identifier)+ rparen +nameTypePairs + : LPAREN nameTypePair (COMMA nameTypePair)+ rparen + ; + +nameTypePair + : identifier (COLON type)? ; // -- assignment statement @@ -349,6 +374,10 @@ multipleAssignmentStatement : variableNames nls ASSIGN nls expression ; +variableNames + : LPAREN identifier (COMMA identifier)+ rparen + ; + assignmentStatement : target=expression nls op=(ASSIGN @@ -577,7 +606,8 @@ formalParameterList ; formalParameter - : DEF? legacyType? identifier (nls ASSIGN nls expression)? + : identifier (COLON type)? (nls ASSIGN nls expression)? + | DEF? legacyType? identifier (nls ASSIGN nls expression)? ; closureWithLabels @@ -649,7 +679,7 @@ namedArg // type : primitiveType - | qualifiedClassName typeArguments? + | qualifiedClassName typeArguments? QUESTION? ; primitiveType diff --git a/modules/nf-lang/src/main/java/nextflow/script/ast/ASTNodeMarker.java b/modules/nf-lang/src/main/java/nextflow/script/ast/ASTNodeMarker.java index 413c46a3cb..d8d7b0b4a2 100644 --- a/modules/nf-lang/src/main/java/nextflow/script/ast/ASTNodeMarker.java +++ b/modules/nf-lang/src/main/java/nextflow/script/ast/ASTNodeMarker.java @@ -45,6 +45,9 @@ public enum ASTNodeMarker { // the MethodNode targeted by a variable expression (PropertyNode) METHOD_VARIABLE_TARGET, + // denotes a nullable type annotation (ClassNode) + NULLABLE, + // the starting quote sequence of a string literal or gstring expression QUOTE_CHAR, diff --git a/modules/nf-lang/src/main/java/nextflow/script/ast/OutputNode.java b/modules/nf-lang/src/main/java/nextflow/script/ast/OutputNode.java index 893947cb4c..13b31fdcf3 100644 --- a/modules/nf-lang/src/main/java/nextflow/script/ast/OutputNode.java +++ b/modules/nf-lang/src/main/java/nextflow/script/ast/OutputNode.java @@ -16,6 +16,7 @@ package nextflow.script.ast; import org.codehaus.groovy.ast.ASTNode; +import org.codehaus.groovy.ast.ClassNode; import org.codehaus.groovy.ast.stmt.Statement; /** @@ -25,10 +26,12 @@ */ public class OutputNode extends ASTNode { public final String name; + public final ClassNode type; public final Statement body; - public OutputNode(String name, Statement body) { + public OutputNode(String name, ClassNode type, Statement body) { this.name = name; + this.type = type; this.body = body; } } diff --git a/modules/nf-lang/src/main/java/nextflow/script/ast/ProcessNode.java b/modules/nf-lang/src/main/java/nextflow/script/ast/ProcessNode.java index ca0ef4ee63..801f6292ed 100644 --- a/modules/nf-lang/src/main/java/nextflow/script/ast/ProcessNode.java +++ b/modules/nf-lang/src/main/java/nextflow/script/ast/ProcessNode.java @@ -19,7 +19,7 @@ import java.util.Optional; import nextflow.script.types.Channel; -import nextflow.script.types.NamedTuple; +import nextflow.script.types.Record; import org.codehaus.groovy.ast.ClassHelper; import org.codehaus.groovy.ast.ClassNode; import org.codehaus.groovy.ast.FieldNode; @@ -66,7 +66,7 @@ private static Parameter[] dummyParams(Statement inputs) { } private static ClassNode dummyReturnType(Statement outputs) { - var cn = new ClassNode(NamedTuple.class); + var cn = new ClassNode(Record.class); asDirectives(outputs) .map(call -> emitName(call)) .filter(name -> name != null) diff --git a/modules/nf-lang/src/main/java/nextflow/script/ast/ScriptVisitorSupport.java b/modules/nf-lang/src/main/java/nextflow/script/ast/ScriptVisitorSupport.java index bbb47dd6b8..e76081c763 100644 --- a/modules/nf-lang/src/main/java/nextflow/script/ast/ScriptVisitorSupport.java +++ b/modules/nf-lang/src/main/java/nextflow/script/ast/ScriptVisitorSupport.java @@ -62,7 +62,6 @@ public void visitParam(ParamNode node) { @Override public void visitWorkflow(WorkflowNode node) { - visit(node.takes); visit(node.main); visit(node.emits); visit(node.publishers); diff --git a/modules/nf-lang/src/main/java/nextflow/script/ast/WorkflowNode.java b/modules/nf-lang/src/main/java/nextflow/script/ast/WorkflowNode.java index 9e3453488d..68206c4ca8 100644 --- a/modules/nf-lang/src/main/java/nextflow/script/ast/WorkflowNode.java +++ b/modules/nf-lang/src/main/java/nextflow/script/ast/WorkflowNode.java @@ -19,7 +19,7 @@ import java.util.Optional; import nextflow.script.types.Channel; -import nextflow.script.types.NamedTuple; +import nextflow.script.types.Record; import org.codehaus.groovy.ast.ClassHelper; import org.codehaus.groovy.ast.ClassNode; import org.codehaus.groovy.ast.FieldNode; @@ -39,14 +39,12 @@ * @author Ben Sherman */ public class WorkflowNode extends MethodNode { - public final Statement takes; public final Statement main; public final Statement emits; public final Statement publishers; - public WorkflowNode(String name, Statement takes, Statement main, Statement emits, Statement publishers) { - super(name, 0, dummyReturnType(emits), dummyParams(takes), ClassNode.EMPTY_ARRAY, EmptyStatement.INSTANCE); - this.takes = takes; + public WorkflowNode(String name, Parameter[] takes, Statement main, Statement emits, Statement publishers) { + super(name, 0, dummyReturnType(emits), takes, ClassNode.EMPTY_ARRAY, EmptyStatement.INSTANCE); this.main = main; this.emits = emits; this.publishers = publishers; @@ -60,15 +58,8 @@ public boolean isCodeSnippet() { return getLineNumber() == -1; } - private static Parameter[] dummyParams(Statement takes) { - return asBlockStatements(takes) - .stream() - .map((stmt) -> new Parameter(ClassHelper.dynamicType(), "")) - .toArray(Parameter[]::new); - } - private static ClassNode dummyReturnType(Statement emits) { - var cn = new ClassNode(NamedTuple.class); + var cn = new ClassNode(Record.class); asBlockStatements(emits).stream() .map(stmt -> ((ExpressionStatement) stmt).getExpression()) .map(emit -> emitName(emit)) diff --git a/modules/nf-lang/src/main/java/nextflow/script/control/ResolveVisitor.java b/modules/nf-lang/src/main/java/nextflow/script/control/ResolveVisitor.java index b70a75833a..3b84cfeb00 100644 --- a/modules/nf-lang/src/main/java/nextflow/script/control/ResolveVisitor.java +++ b/modules/nf-lang/src/main/java/nextflow/script/control/ResolveVisitor.java @@ -22,6 +22,7 @@ import groovy.lang.Tuple2; import nextflow.script.ast.ASTNodeMarker; +import nextflow.script.types.Bag; import org.codehaus.groovy.GroovyBugError; import org.codehaus.groovy.ast.ASTNode; import org.codehaus.groovy.ast.ClassCodeExpressionTransformer; @@ -60,9 +61,16 @@ */ public class ResolveVisitor extends ClassCodeExpressionTransformer { - public static final String[] DEFAULT_PACKAGE_PREFIXES = { "java.lang.", "java.util.", "java.io.", "java.net.", "groovy.lang.", "groovy.util." }; - - public static final String[] EMPTY_STRING_ARRAY = new String[0]; + public static final ClassNode[] STANDARD_TYPES = { + ClassHelper.makeCached(Bag.class), + ClassHelper.Boolean_TYPE, + ClassHelper.Integer_TYPE, + ClassHelper.Number_TYPE, + ClassHelper.STRING_TYPE, + ClassHelper.LIST_TYPE, + ClassHelper.MAP_TYPE, + ClassHelper.SET_TYPE + }; private SourceUnit sourceUnit; @@ -113,12 +121,16 @@ public boolean resolve(ClassNode type) { return true; if( type.isResolved() ) return true; - if( resolveFromModule(type) ) + if( !type.hasPackageName() && resolveFromModule(type) ) + return true; + if( !type.hasPackageName() && resolveFromStandardTypes(type) ) return true; if( resolveFromLibImports(type) ) return true; if( !type.hasPackageName() && resolveFromDefaultImports(type) ) return true; + if( !type.hasPackageName() && resolveFromGroovyImports(type) ) + return true; return resolveFromClassResolver(type.getName()) != null; } @@ -156,10 +168,19 @@ protected boolean resolveFromModule(ClassNode type) { return false; } + protected boolean resolveFromStandardTypes(ClassNode type) { + for( var cn : STANDARD_TYPES ) { + if( cn.getNameWithoutPackage().equals(type.getName()) ) { + type.setRedirect(cn); + return true; + } + } + return false; + } + protected boolean resolveFromLibImports(ClassNode type) { - var name = type.getName(); for( var cn : libImports ) { - if( name.equals(cn.getName()) ) { + if( cn.getName().equals(type.getName()) ) { type.setRedirect(cn); return true; } @@ -168,22 +189,29 @@ protected boolean resolveFromLibImports(ClassNode type) { } protected boolean resolveFromDefaultImports(ClassNode type) { - // resolve from script imports - var typeName = type.getName(); for( var cn : defaultImports ) { - if( typeName.equals(cn.getNameWithoutPackage()) ) { + if( cn.getNameWithoutPackage().equals(type.getName()) ) { type.setRedirect(cn); return true; } } - // resolve from default imports cache + return false; + } + + private static final String[] DEFAULT_PACKAGE_PREFIXES = { "java.lang.", "java.util.", "java.io.", "java.net.", "groovy.lang.", "groovy.util." }; + + private static final String[] EMPTY_STRING_ARRAY = new String[0]; + + protected boolean resolveFromGroovyImports(ClassNode type) { + var typeName = type.getName(); + // resolve from Groovy imports cache var packagePrefixSet = DEFAULT_IMPORT_CLASS_AND_PACKAGES_CACHE.get(typeName); if( packagePrefixSet != null ) { - if( resolveFromDefaultImports(type, packagePrefixSet.toArray(EMPTY_STRING_ARRAY)) ) + if( resolveFromGroovyImports(type, packagePrefixSet.toArray(EMPTY_STRING_ARRAY)) ) return true; } - // resolve from default imports - if( resolveFromDefaultImports(type, DEFAULT_PACKAGE_PREFIXES) ) { + // resolve from Groovy imports + if( resolveFromGroovyImports(type, DEFAULT_PACKAGE_PREFIXES) ) { return true; } if( "BigInteger".equals(typeName) ) { @@ -202,7 +230,7 @@ protected boolean resolveFromDefaultImports(ClassNode type) { DEFAULT_IMPORT_CLASS_AND_PACKAGES_CACHE.putAll(VMPluginFactory.getPlugin().getDefaultImportClasses(DEFAULT_PACKAGE_PREFIXES)); } - protected boolean resolveFromDefaultImports(ClassNode type, String[] packagePrefixes) { + protected boolean resolveFromGroovyImports(ClassNode type, String[] packagePrefixes) { var typeName = type.getName(); for( var packagePrefix : packagePrefixes ) { var redirect = resolveFromClassResolver(packagePrefix + typeName); diff --git a/modules/nf-lang/src/main/java/nextflow/script/control/ScriptResolveVisitor.java b/modules/nf-lang/src/main/java/nextflow/script/control/ScriptResolveVisitor.java index 67f6813db7..3469caa4a9 100644 --- a/modules/nf-lang/src/main/java/nextflow/script/control/ScriptResolveVisitor.java +++ b/modules/nf-lang/src/main/java/nextflow/script/control/ScriptResolveVisitor.java @@ -85,6 +85,8 @@ public void visitParam(ParamNode node) { @Override public void visitWorkflow(WorkflowNode node) { + for( var take : node.getParameters() ) + resolver.resolveOrFail(take.getType(), take); resolver.visit(node.main); resolver.visit(node.emits); resolver.visit(node.publishers); diff --git a/modules/nf-lang/src/main/java/nextflow/script/control/ScriptToGroovyVisitor.java b/modules/nf-lang/src/main/java/nextflow/script/control/ScriptToGroovyVisitor.java index 08aa004029..83088ce2f0 100644 --- a/modules/nf-lang/src/main/java/nextflow/script/control/ScriptToGroovyVisitor.java +++ b/modules/nf-lang/src/main/java/nextflow/script/control/ScriptToGroovyVisitor.java @@ -15,6 +15,7 @@ */ package nextflow.script.control; +import java.util.Arrays; import java.util.List; import java.util.Set; import java.util.stream.Collectors; @@ -31,6 +32,7 @@ import nextflow.script.ast.WorkflowNode; import org.codehaus.groovy.ast.ASTNode; import org.codehaus.groovy.ast.CodeVisitorSupport; +import org.codehaus.groovy.ast.Parameter; import org.codehaus.groovy.ast.VariableScope; import org.codehaus.groovy.ast.expr.ArgumentListExpression; import org.codehaus.groovy.ast.expr.BinaryExpression; @@ -124,7 +126,6 @@ public void visitParam(ParamNode node) { @Override public void visitWorkflow(WorkflowNode node) { - visitWorkflowTakes(node.takes); visit(node.main); visitWorkflowEmits(node.emits, node.main); visitWorkflowPublishers(node.publishers, node.main); @@ -138,7 +139,7 @@ public void visitWorkflow(WorkflowNode node) { ) )); var closure = closureX(null, block(new VariableScope(), List.of( - node.takes, + workflowTakes(node.getParameters()), node.emits, bodyDef ))); @@ -149,12 +150,13 @@ public void visitWorkflow(WorkflowNode node) { moduleNode.addStatement(result); } - private void visitWorkflowTakes(Statement takes) { - for( var stmt : asBlockStatements(takes) ) { - var es = (ExpressionStatement)stmt; - var take = (VariableExpression)es.getExpression(); - es.setExpression(callThisX("_take_", args(constX(take.getName())))); - } + private Statement workflowTakes(Parameter[] takes) { + var statements = Arrays.stream(takes) + .map((take) -> + stmt(callThisX("_take_", args(constX(take.getName())))) + ) + .toList(); + return block(null, statements); } private void visitWorkflowEmits(Statement emits, Statement main) { diff --git a/modules/nf-lang/src/main/java/nextflow/script/control/VariableScopeVisitor.java b/modules/nf-lang/src/main/java/nextflow/script/control/VariableScopeVisitor.java index d46da88e28..3ac0252795 100644 --- a/modules/nf-lang/src/main/java/nextflow/script/control/VariableScopeVisitor.java +++ b/modules/nf-lang/src/main/java/nextflow/script/control/VariableScopeVisitor.java @@ -176,7 +176,8 @@ public void visitWorkflow(WorkflowNode node) { currentDefinition = node; node.setVariableScope(currentScope()); - declareWorkflowInputs(node.takes); + for( var take : node.getParameters() ) + vsc.declare(take, take); visit(node.main); if( node.main instanceof BlockStatement block ) @@ -189,15 +190,6 @@ public void visitWorkflow(WorkflowNode node) { vsc.popScope(); } - private void declareWorkflowInputs(Statement takes) { - for( var stmt : asBlockStatements(takes) ) { - var ve = asVarX(stmt); - if( ve == null ) - continue; - vsc.declare(ve); - } - } - private void copyVariableScope(VariableScope source) { for( var it = source.getDeclaredVariablesIterator(); it.hasNext(); ) { var variable = it.next(); @@ -345,7 +337,7 @@ public void visitFunction(FunctionNode node) { for( var parameter : node.getParameters() ) { if( parameter.hasInitialExpression() ) visit(parameter.getInitialExpression()); - vsc.declare(parameter, node); + vsc.declare(parameter, parameter); } visit(node.getCode()); diff --git a/modules/nf-lang/src/main/java/nextflow/script/dsl/ScriptDsl.java b/modules/nf-lang/src/main/java/nextflow/script/dsl/ScriptDsl.java index e460e84784..4e26291904 100644 --- a/modules/nf-lang/src/main/java/nextflow/script/dsl/ScriptDsl.java +++ b/modules/nf-lang/src/main/java/nextflow/script/dsl/ScriptDsl.java @@ -25,6 +25,7 @@ import nextflow.script.namespaces.LogNamespace; import nextflow.script.namespaces.NextflowNamespace; import nextflow.script.namespaces.WorkflowNamespace; +import nextflow.script.types.Tuple; /** * The built-in namespaces, constants, and functions in a script. @@ -180,6 +181,6 @@ The directory where a module script is located (equivalent to `projectDir` if us @Description(""" Create a tuple from the given arguments. """) - List tuple(Object... args); + Tuple tuple(Object... args); } diff --git a/modules/nf-lang/src/main/java/nextflow/script/formatter/Formatter.java b/modules/nf-lang/src/main/java/nextflow/script/formatter/Formatter.java index 7f83cd24b1..3654b7e028 100644 --- a/modules/nf-lang/src/main/java/nextflow/script/formatter/Formatter.java +++ b/modules/nf-lang/src/main/java/nextflow/script/formatter/Formatter.java @@ -24,6 +24,7 @@ import org.codehaus.groovy.ast.ClassNode; import org.codehaus.groovy.ast.CodeVisitorSupport; import org.codehaus.groovy.ast.Parameter; +import org.codehaus.groovy.ast.Variable; import org.codehaus.groovy.ast.expr.BinaryExpression; import org.codehaus.groovy.ast.expr.BitwiseNegationExpression; import org.codehaus.groovy.ast.expr.CastExpression; @@ -254,12 +255,11 @@ public void visitCatchStatement(CatchStatement node) { append("catch ("); var variable = node.getVariable(); - var type = variable.getType(); - if( !ClassHelper.isObjectType(type) ) { - append(type.getNameWithoutPackage()); - append(' '); - } append(variable.getName()); + if( hasType(variable) ) { + append(": "); + append(variable.getType().getNameWithoutPackage()); + } append(") {\n"); incIndent(); @@ -500,11 +500,11 @@ else if( code.getStatements().size() == 1 && code.getStatements().get(0) instanc public void visitParameters(Parameter[] parameters) { for( int i = 0; i < parameters.length; i++ ) { var param = parameters[i]; - if( isLegacyType(param.getType()) ) { + append(param.getName()); + if( hasType(param) ) { + append(": "); visitTypeAnnotation(param.getType()); - append(' '); } - append(param.getName()); if( param.hasInitialExpression() ) { append(" = "); visit(param.getInitialExpression()); @@ -649,21 +649,19 @@ public void visitClassExpression(ClassExpression node) { } public void visitTypeAnnotation(ClassNode type) { - if( isLegacyType(type) ) { + if( isLegacyType(type) ) append(type.getNodeMetaData(ASTNodeMarker.LEGACY_TYPE)); - return; - } - - append(nextflow.script.types.Types.getName(type)); + else + append(nextflow.script.types.Types.getName(type)); } @Override public void visitVariableExpression(VariableExpression node) { - if( inVariableDeclaration && isLegacyType(node.getType()) ) { + append(node.getText()); + if( inVariableDeclaration && hasType(node) ) { + append(": "); visitTypeAnnotation(node.getType()); - append(' '); } - append(node.getText()); } @Override @@ -713,6 +711,14 @@ private static boolean hasTrailingComma(Expression node) { return node.getNodeMetaData(ASTNodeMarker.TRAILING_COMMA) != null; } + public static boolean hasType(ClassNode type) { + return !ClassHelper.isDynamicTyped(type) || isLegacyType(type); + } + + public static boolean hasType(Variable variable) { + return !variable.isDynamicTyped() || isLegacyType(variable.getType()); + } + public static boolean isLegacyType(ClassNode cn) { return cn.getNodeMetaData(ASTNodeMarker.LEGACY_TYPE) != null; } diff --git a/modules/nf-lang/src/main/java/nextflow/script/formatter/ScriptFormattingVisitor.java b/modules/nf-lang/src/main/java/nextflow/script/formatter/ScriptFormattingVisitor.java index 5ed37052d0..fe2a0d3c8b 100644 --- a/modules/nf-lang/src/main/java/nextflow/script/formatter/ScriptFormattingVisitor.java +++ b/modules/nf-lang/src/main/java/nextflow/script/formatter/ScriptFormattingVisitor.java @@ -15,6 +15,7 @@ */ package nextflow.script.formatter; +import java.util.Arrays; import java.util.Comparator; import java.util.List; @@ -31,7 +32,9 @@ import nextflow.script.ast.ScriptVisitorSupport; import nextflow.script.ast.WorkflowNode; import org.codehaus.groovy.ast.ClassNode; +import org.codehaus.groovy.ast.Parameter; import org.codehaus.groovy.ast.expr.EmptyExpression; +import org.codehaus.groovy.ast.expr.Expression; import org.codehaus.groovy.ast.expr.PropertyExpression; import org.codehaus.groovy.ast.expr.VariableExpression; import org.codehaus.groovy.ast.stmt.BlockStatement; @@ -217,14 +220,15 @@ public void visitWorkflow(WorkflowNode node) { } fmt.append(" {\n"); fmt.incIndent(); - if( node.takes instanceof BlockStatement ) { + var takes = node.getParameters(); + if( takes.length > 0 ) { fmt.appendIndent(); fmt.append("take:\n"); - visitWorkflowTakes(asBlockStatements(node.takes)); + visitWorkflowTakes(takes); fmt.appendNewLine(); } if( node.main instanceof BlockStatement ) { - if( node.takes instanceof BlockStatement || node.emits instanceof BlockStatement || node.publishers instanceof BlockStatement ) { + if( takes.length > 0 || node.emits instanceof BlockStatement || node.publishers instanceof BlockStatement ) { fmt.appendIndent(); fmt.append("main:\n"); } @@ -240,92 +244,119 @@ public void visitWorkflow(WorkflowNode node) { fmt.appendNewLine(); fmt.appendIndent(); fmt.append("publish:\n"); - fmt.visit(node.publishers); + visitWorkflowPublishers(asBlockStatements(node.publishers)); } fmt.decIndent(); fmt.append("}\n"); } - protected void visitWorkflowTakes(List takes) { + private void visitWorkflowTakes(Parameter[] takes) { var alignmentWidth = options.harshilAlignment() - ? getMaxParameterWidth(takes) + ? maxParameterWidth(takes) : 0; - for( var stmt : takes ) { - var ve = asVarX(stmt); + for( var take : takes ) { fmt.appendIndent(); - fmt.visit(ve); - if( fmt.hasTrailingComment(stmt) ) { + fmt.append(take.getName()); + if( fmt.hasType(take) ) { if( alignmentWidth > 0 ) { - var padding = alignmentWidth - ve.getName().length(); + var padding = alignmentWidth - take.getName().length() + 1; fmt.append(" ".repeat(padding)); } - fmt.appendTrailingComment(stmt); + fmt.append(": "); + fmt.visitTypeAnnotation(take.getType()); } + fmt.appendTrailingComment(take); fmt.appendNewLine(); } } - protected void visitWorkflowEmits(List emits) { + private int maxParameterWidth(Parameter[] parameters) { + return Arrays.stream(parameters) + .map(param -> param.getName().length()) + .max(Integer::compare).orElse(0); + } + + private void visitWorkflowEmits(List emits) { var alignmentWidth = options.harshilAlignment() - ? getMaxParameterWidth(emits) + ? maxParameterWidth(emits) : 0; for( var stmt : emits ) { var stmtX = (ExpressionStatement)stmt; var emit = stmtX.getExpression(); - if( emit instanceof AssignmentExpression assign ) { - var ve = (VariableExpression)assign.getLeftExpression(); + var target = + emit instanceof AssignmentExpression ae ? (VariableExpression)ae.getLeftExpression() : + emit instanceof VariableExpression ve ? ve : + null; + var source = + emit instanceof AssignmentExpression ae ? ae.getRightExpression() : + null; + + if( target != null ) { fmt.appendIndent(); - fmt.visit(ve); - if( alignmentWidth > 0 ) { - var padding = alignmentWidth - ve.getName().length(); - fmt.append(" ".repeat(padding)); - } - fmt.append(" = "); - fmt.visit(assign.getRightExpression()); + visitEmitAssignment(target, source, alignmentWidth); fmt.appendTrailingComment(stmt); fmt.appendNewLine(); } - else if( emit instanceof VariableExpression ve ) { - fmt.appendIndent(); - fmt.visit(ve); - if( fmt.hasTrailingComment(stmt) ) { - if( alignmentWidth > 0 ) { - var padding = alignmentWidth - ve.getName().length(); - fmt.append(" ".repeat(padding)); - } - fmt.appendTrailingComment(stmt); - } - fmt.appendNewLine(); - } else { fmt.visit(stmt); } } } - protected int getMaxParameterWidth(List statements) { + private void visitWorkflowPublishers(List publishers) { + var alignmentWidth = options.harshilAlignment() + ? maxParameterWidth(publishers) + : 0; + + for( var stmt : publishers ) { + var stmtX = (ExpressionStatement)stmt; + var emit = (AssignmentExpression)stmtX.getExpression(); + var target = (VariableExpression)emit.getLeftExpression(); + var source = emit.getRightExpression(); + + fmt.appendIndent(); + visitEmitAssignment(target, source, alignmentWidth); + fmt.appendTrailingComment(stmt); + fmt.appendNewLine(); + } + } + + private int maxParameterWidth(List statements) { if( statements.size() == 1 ) return 0; - int maxWidth = 0; - for( var stmt : statements ) { - var stmtX = (ExpressionStatement)stmt; - var emit = stmtX.getExpression(); - int width = 0; - if( emit instanceof VariableExpression ve ) { - width = ve.getName().length(); - } - else if( emit instanceof AssignmentExpression assign ) { - var target = (VariableExpression)assign.getLeftExpression(); - width = target.getName().length(); - } + return statements.stream() + .map((stmt) -> { + var stmtX = (ExpressionStatement)stmt; + var emit = stmtX.getExpression(); + if( emit instanceof VariableExpression ve ) { + return ve.getName().length(); + } + if( emit instanceof AssignmentExpression assign ) { + var target = (VariableExpression)assign.getLeftExpression(); + return target.getName().length(); + } + return 0; + }) + .max(Integer::compare).orElse(0); + } - if( maxWidth < width ) - maxWidth = width; + private void visitEmitAssignment(VariableExpression target, Expression source, int alignmentWidth) { + fmt.append(target.getText()); + if( fmt.hasType(target) ) { + if( alignmentWidth > 0 ) { + var padding = alignmentWidth - target.getName().length() + 1; + fmt.append(" ".repeat(padding)); + } + fmt.append(": "); + fmt.visitTypeAnnotation(target.getType()); + } + if( source != null ) { + fmt.append(" = "); + fmt.visit(source); } - return maxWidth; } @Override @@ -384,14 +415,15 @@ private void visitProcessOutputs(Statement outputs) { public void visitFunction(FunctionNode node) { fmt.appendLeadingComments(node); fmt.append("def "); - if( Formatter.isLegacyType(node.getReturnType()) ) { - fmt.visitTypeAnnotation(node.getReturnType()); - fmt.append(' '); - } fmt.append(node.getName()); fmt.append('('); fmt.visitParameters(node.getParameters()); - fmt.append(") {\n"); + fmt.append(')'); + if( fmt.hasType(node.getReturnType()) ) { + fmt.append(" -> "); + fmt.visitTypeAnnotation(node.getReturnType()); + } + fmt.append(" {\n"); fmt.incIndent(); fmt.visit(node.getCode()); fmt.decIndent(); @@ -430,6 +462,10 @@ public void visitOutput(OutputNode node) { fmt.appendLeadingComments(node); fmt.appendIndent(); fmt.append(node.name); + if( fmt.hasType(node.type) ) { + fmt.append(": "); + fmt.visitTypeAnnotation(node.type); + } fmt.append(" {\n"); fmt.incIndent(); visitOutputBody((BlockStatement) node.body); diff --git a/modules/nf-lang/src/main/java/nextflow/script/parser/ScriptAstBuilder.java b/modules/nf-lang/src/main/java/nextflow/script/parser/ScriptAstBuilder.java index e66fe36734..fc1e61e05a 100644 --- a/modules/nf-lang/src/main/java/nextflow/script/parser/ScriptAstBuilder.java +++ b/modules/nf-lang/src/main/java/nextflow/script/parser/ScriptAstBuilder.java @@ -507,7 +507,7 @@ private WorkflowNode workflowDef(WorkflowDefContext ctx) { var name = ctx.name != null ? ctx.name.getText() : null; if( ctx.body == null ) { - var result = ast( new WorkflowNode(name, null, null, null, null), ctx ); + var result = ast( new WorkflowNode(name, Parameter.EMPTY_ARRAY, null, null, null), ctx ); groovydocManager.handle(result, ctx); return result; } @@ -522,10 +522,10 @@ private WorkflowNode workflowDef(WorkflowDefContext ctx) { ); if( name == null ) { - if( takes instanceof BlockStatement ) - collectSyntaxError(new SyntaxException("Entry workflow cannot have a take section", takes)); - if( emits instanceof BlockStatement ) - collectSyntaxError(new SyntaxException("Entry workflow cannot have an emit section", emits)); + if( ctx.body.TAKE() != null ) + collectSyntaxError(new SyntaxException("Entry workflow cannot have a take section", ast( new EmptyStatement(), ctx.body.TAKE() ))); + if( ctx.body.EMIT() != null ) + collectSyntaxError(new SyntaxException("Entry workflow cannot have an emit section", ast( new EmptyStatement(), ctx.body.EMIT() ))); } if( name != null ) { if( publishers instanceof BlockStatement ) @@ -538,24 +538,31 @@ private WorkflowNode workflowDef(WorkflowDefContext ctx) { } private WorkflowNode workflowDef(BlockStatement main) { - var takes = EmptyStatement.INSTANCE; + var takes = Parameter.EMPTY_ARRAY; var emits = EmptyStatement.INSTANCE; var publishers = EmptyStatement.INSTANCE; return new WorkflowNode(null, takes, main, emits, publishers); } - private Statement workflowTakes(WorkflowTakesContext ctx) { + private Parameter[] workflowTakes(WorkflowTakesContext ctx) { if( ctx == null ) - return EmptyStatement.INSTANCE; + return Parameter.EMPTY_ARRAY; - var statements = ctx.identifier().stream() + return ctx.workflowTake().stream() .map(this::workflowTake) - .toList(); - return ast( block(null, statements), ctx ); + .filter(take -> take != null) + .toArray(Parameter[]::new); } - private Statement workflowTake(IdentifierContext ctx) { - var result = ast( stmt(variableName(ctx)), ctx ); + private Parameter workflowTake(WorkflowTakeContext ctx) { + if( ctx.statement() != null ) { + collectSyntaxError(new SyntaxException("Invalid workflow take", ast( new EmptyStatement(), ctx.statement() ))); + return null; + } + var type = type(ctx.type()); + var name = identifier(ctx.identifier()); + var result = ast( param(type, name), ctx ); + checkInvalidVarName(name, result); saveTrailingComment(result, ctx); return result; } @@ -564,7 +571,7 @@ private Statement workflowEmits(WorkflowEmitsContext ctx) { if( ctx == null ) return EmptyStatement.INSTANCE; - var statements = ctx.statement().stream() + var statements = ctx.workflowEmit().stream() .map(this::workflowEmit) .filter(stmt -> stmt != null) .toList(); @@ -575,11 +582,23 @@ private Statement workflowEmits(WorkflowEmitsContext ctx) { return result; } - private Statement workflowEmit(StatementContext ctx) { - var result = statement(ctx); - if( !(result instanceof ExpressionStatement) ) { - collectSyntaxError(new SyntaxException("Invalid workflow emit -- must be a name, assignment, or expression", result)); - return null; + private Statement workflowEmit(WorkflowEmitContext ctx) { + Statement result; + if( ctx.statement() != null ) { + result = statement(ctx.statement()); + if( !(result instanceof ExpressionStatement) ) { + collectSyntaxError(new SyntaxException("Invalid workflow emit -- must be a name, assignment, or expression", result)); + return null; + } + } + else if( ctx.expression() != null ) { + var target = nameTypePair(ctx.nameTypePair()); + var source = expression(ctx.expression()); + result = stmt(ast( new AssignmentExpression(target, source), ctx )); + } + else { + var target = nameTypePair(ctx.nameTypePair()); + result = stmt(target); } saveTrailingComment(result, ctx); return result; @@ -597,24 +616,29 @@ private Statement workflowPublishers(WorkflowPublishersContext ctx) { if( ctx == null ) return EmptyStatement.INSTANCE; - var statements = ctx.statement().stream() - .map(this::statement) - .map(this::checkWorkflowPublisher) + var statements = ctx.workflowEmit().stream() + .map(this::workflowPublisher) .filter(stmt -> stmt != null) .toList(); return ast( block(null, statements), ctx ); } - private Statement checkWorkflowPublisher(Statement stmt) { - var valid = stmt instanceof ExpressionStatement es - && es.getExpression() instanceof BinaryExpression be - && be.getLeftExpression() instanceof VariableExpression - && be.getOperation().getType() == Types.ASSIGN; - if( !valid ) { - collectSyntaxError(new SyntaxException("Invalid workflow publish statement", stmt)); + private Statement workflowPublisher(WorkflowEmitContext ctx) { + if( ctx.statement() != null ) { + collectSyntaxError(new SyntaxException("Invalid workflow publish statement -- must be an assignment", ast( new EmptyStatement(), ctx.statement() ))); return null; } - return stmt; + var target = nameTypePair(ctx.nameTypePair()); + Statement result; + if( ctx.expression() != null ) { + var source = expression(ctx.expression()); + result = stmt(ast( new AssignmentExpression(target, source), ctx )); + } + else { + result = stmt(target); + } + saveTrailingComment(result, ctx); + return result; } private OutputBlockNode outputDef(OutputDefContext ctx) { @@ -637,16 +661,19 @@ private OutputNode outputDeclaration(OutputDeclarationContext ctx) { return null; } var name = identifier(ctx.identifier()); + var type = type(ctx.type()); var body = blockStatements(ctx.blockStatements()); - var result = new OutputNode(name, body); + var result = new OutputNode(name, type, body); checkInvalidVarName(name, result); return result; } private FunctionNode functionDef(FunctionDefContext ctx) { var name = identifier(ctx.identifier()); - var returnType = legacyType(ctx.legacyType()); var params = Optional.ofNullable(formalParameterList(ctx.formalParameterList())).orElse(Parameter.EMPTY_ARRAY); + var returnType = ctx.type() != null + ? type(ctx.type()) + : legacyType(ctx.legacyType()); var code = blockStatements(ctx.blockStatements()); var result = ast( new FunctionNode(name, returnType, params, code), ctx ); @@ -741,14 +768,23 @@ private Statement tryCatchStatement(TryCatchStatementContext ctx) { } private List catchClause(CatchClauseContext ctx) { + var variables = catchVariables(ctx.catchVariable()); + return variables.stream() + .map((variable) -> { + var code = statementOrBlock(ctx.statementOrBlock()); + return ast( new CatchStatement(variable, code), ctx ); + }) + .toList(); + } + + private List catchVariables(CatchVariableContext ctx) { var types = catchTypes(ctx.catchTypes()); return types.stream() - .map(type -> { + .map((type) -> { var name = identifier(ctx.identifier()); var variable = ast( param(type, name), ctx.identifier() ); checkInvalidVarName(name, variable); - var code = statementOrBlock(ctx.statementOrBlock()); - return ast( new CatchStatement(variable, code), ctx ); + return variable; }) .toList(); } @@ -784,17 +820,25 @@ private Statement assertStatement(AssertStatementContext ctx) { } private Statement variableDeclaration(VariableDeclarationContext ctx) { - if( ctx.variableNames() != null ) { + if( ctx.nameTypePairs() != null ) { // multiple assignment - var variables = ctx.variableNames().identifier().stream() - .map(ident -> (Expression) variableName(ident)) + var variables = ctx.nameTypePairs().nameTypePair().stream() + .map(this::nameTypePair) .toList(); var target = new ArgumentListExpression(variables); var initializer = expression(ctx.initializer); return stmt(ast( declX(target, initializer), ctx )); } - else { + else if( ctx.nameTypePair() != null ) { // single assignment + var target = nameTypePair(ctx.nameTypePair()); + var initializer = ctx.initializer != null + ? expression(ctx.initializer) + : EmptyExpression.INSTANCE; + return stmt(ast( declX(target, initializer), ctx )); + } + else { + // single assignment (legacy type) var target = variableName(ctx.identifier()); target.setType(legacyType(ctx.legacyType())); var initializer = ctx.initializer != null @@ -804,11 +848,12 @@ private Statement variableDeclaration(VariableDeclarationContext ctx) { } } - private Expression variableNames(VariableNamesContext ctx) { - var vars = ctx.identifier().stream() - .map(this::variableName) - .toList(); - return ast( new TupleExpression(vars), ctx ); + private Expression nameTypePair(NameTypePairContext ctx) { + var name = identifier(ctx.identifier()); + var type = type(ctx.type()); + var result = ast( varX(name, type), ctx ); + checkInvalidVarName(name, result); + return result; } private Expression variableName(IdentifierContext ctx) { @@ -841,6 +886,13 @@ private Statement assignment(MultipleAssignmentStatementContext ctx) { return stmt(ast( new AssignmentExpression(target, source), ctx )); } + private Expression variableNames(VariableNamesContext ctx) { + var vars = ctx.identifier().stream() + .map(this::variableName) + .toList(); + return ast( new TupleExpression(vars), ctx ); + } + private Statement assignment(AssignmentStatementContext ctx) { var target = expression(ctx.target); if( target instanceof VariableExpression && isInsideParentheses(target) ) { @@ -1554,7 +1606,9 @@ private Parameter[] formalParameterList(FormalParameterListContext ctx) { } private Parameter formalParameter(FormalParameterContext ctx) { - var type = legacyType(ctx.legacyType()); + var type = ctx.type() != null + ? type(ctx.type()) + : legacyType(ctx.legacyType()); var name = identifier(ctx.identifier()); var defaultValue = ctx.expression() != null ? expression(ctx.expression()) @@ -1582,22 +1636,22 @@ private org.codehaus.groovy.syntax.Token token(Token token, int cardinality) { } private ClassNode createdName(CreatedNameContext ctx) { + if( ctx.primitiveType() != null ) + return primitiveType(ctx.primitiveType()); + if( ctx.qualifiedClassName() != null ) { - var classNode = qualifiedClassName(ctx.qualifiedClassName()); + var result = qualifiedClassName(ctx.qualifiedClassName()); if( ctx.typeArguments() != null ) - classNode.setGenericsTypes( typeArguments(ctx.typeArguments()) ); - return classNode; + result.setGenericsTypes( typeArguments(ctx.typeArguments()) ); + return result; } - if( ctx.primitiveType() != null ) - return primitiveType(ctx.primitiveType()); - throw createParsingFailedException("Unrecognized created name: " + ctx.getText(), ctx); } private ClassNode primitiveType(PrimitiveTypeContext ctx) { - var classNode = ClassHelper.make(ctx.getText()).getPlainNodeReference(false); - return ast( classNode, ctx ); + var result = ClassHelper.make(ctx.getText()).getPlainNodeReference(false); + return ast( result, ctx ); } private ClassNode qualifiedClassName(QualifiedClassNameContext ctx) { @@ -1606,17 +1660,17 @@ private ClassNode qualifiedClassName(QualifiedClassNameContext ctx) { private ClassNode qualifiedClassName(QualifiedClassNameContext ctx, boolean allowProxy) { var text = ctx.getText(); - var classNode = ClassHelper.make(text); + var result = ClassHelper.make(text); if( text.contains(".") ) - classNode.putNodeMetaData(ASTNodeMarker.FULLY_QUALIFIED, true); + result.putNodeMetaData(ASTNodeMarker.FULLY_QUALIFIED, true); - if( classNode.isUsingGenerics() && allowProxy ) { - var proxy = ClassHelper.makeWithoutCaching(classNode.getName()); - proxy.setRedirect(classNode); + if( result.isUsingGenerics() && allowProxy ) { + var proxy = ClassHelper.makeWithoutCaching(result.getName()); + proxy.setRedirect(result); return proxy; } - return ast( classNode, ctx ); + return ast( result, ctx ); } private ClassNode type(TypeContext ctx) { @@ -1627,16 +1681,18 @@ private ClassNode type(TypeContext ctx, boolean allowProxy) { if( ctx == null ) return ClassHelper.dynamicType(); + if( ctx.primitiveType() != null ) + return primitiveType(ctx.primitiveType()); + if( ctx.qualifiedClassName() != null ) { - var classNode = qualifiedClassName(ctx.qualifiedClassName(), allowProxy); + var result = qualifiedClassName(ctx.qualifiedClassName(), allowProxy); if( ctx.typeArguments() != null ) - classNode.setGenericsTypes( typeArguments(ctx.typeArguments()) ); - return classNode; + result.setGenericsTypes( typeArguments(ctx.typeArguments()) ); + if( ctx.QUESTION() != null ) + result.putNodeMetaData(ASTNodeMarker.NULLABLE, Boolean.TRUE); + return result; } - if( ctx.primitiveType() != null ) - return primitiveType(ctx.primitiveType()); - throw createParsingFailedException("Unrecognized type: " + ctx.getText(), ctx); } diff --git a/modules/nf-lang/src/main/java/nextflow/script/types/Bag.java b/modules/nf-lang/src/main/java/nextflow/script/types/Bag.java index 4590e0d6e4..2721ece406 100644 --- a/modules/nf-lang/src/main/java/nextflow/script/types/Bag.java +++ b/modules/nf-lang/src/main/java/nextflow/script/types/Bag.java @@ -17,7 +17,5 @@ import java.util.Collection; -import nextflow.script.dsl.Description; - public interface Bag extends Collection { } diff --git a/modules/nf-lang/src/main/java/nextflow/script/types/NamedTuple.java b/modules/nf-lang/src/main/java/nextflow/script/types/Record.java similarity index 87% rename from modules/nf-lang/src/main/java/nextflow/script/types/NamedTuple.java rename to modules/nf-lang/src/main/java/nextflow/script/types/Record.java index d77cef022e..0e468e4d12 100644 --- a/modules/nf-lang/src/main/java/nextflow/script/types/NamedTuple.java +++ b/modules/nf-lang/src/main/java/nextflow/script/types/Record.java @@ -16,7 +16,7 @@ package nextflow.script.types; /** - * Placeholder type used to model process and workflow outputs. + * Placeholder type used to model records. */ -public class NamedTuple { +public interface Record { } diff --git a/modules/nf-commons/src/main/nextflow/util/Bag.groovy b/modules/nf-lang/src/main/java/nextflow/script/types/Tuple.java similarity index 70% rename from modules/nf-commons/src/main/nextflow/util/Bag.groovy rename to modules/nf-lang/src/main/java/nextflow/script/types/Tuple.java index 490c39296f..3695400ec4 100644 --- a/modules/nf-commons/src/main/nextflow/util/Bag.groovy +++ b/modules/nf-lang/src/main/java/nextflow/script/types/Tuple.java @@ -1,5 +1,5 @@ /* - * Copyright 2013-2024, Seqera Labs + * Copyright 2024-2025, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -13,13 +13,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - -package nextflow.util +package nextflow.script.types; /** - * Marker interface that define that the collection does not care about items order - * - * @author Paolo Di Tommaso + * Placeholder type used to model tuples. */ -interface Bag extends Collection { +public interface Tuple { } diff --git a/modules/nf-lang/src/main/java/nextflow/script/types/Types.java b/modules/nf-lang/src/main/java/nextflow/script/types/Types.java index 84000f0590..df72bcd45f 100644 --- a/modules/nf-lang/src/main/java/nextflow/script/types/Types.java +++ b/modules/nf-lang/src/main/java/nextflow/script/types/Types.java @@ -45,7 +45,8 @@ public class Types { new ClassNode(Channel.class), new ClassNode(Duration.class), new ClassNode(MemoryUnit.class), - new ClassNode(Path.class) + new ClassNode(Path.class), + new ClassNode(VersionNumber.class) ); public static final List DEFAULT_CONFIG_IMPORTS = List.of( @@ -78,7 +79,7 @@ public static boolean isAssignableFrom(ClassNode target, ClassNode source) { return true; if( target.equals(source) ) return true; - return isAssignableFrom(target.getTypeClass(), source.getTypeClass()); + return hasTypeClass(target) && hasTypeClass(source) && isAssignableFrom(target.getTypeClass(), source.getTypeClass()); } public static boolean isAssignableFrom(Class target, Class source) { @@ -183,6 +184,9 @@ else if( hasTypeClass(type) ) builder.append('>'); } + if( type.getNodeMetaData(ASTNodeMarker.NULLABLE) != null ) + builder.append('?'); + return builder.toString(); } diff --git a/modules/nf-lang/src/main/java/nextflow/script/types/shim/String.java b/modules/nf-lang/src/main/java/nextflow/script/types/shim/String.java index 036714bb1e..511fa324da 100644 --- a/modules/nf-lang/src/main/java/nextflow/script/types/shim/String.java +++ b/modules/nf-lang/src/main/java/nextflow/script/types/shim/String.java @@ -56,15 +56,25 @@ public interface String { boolean isEmpty(); @Description(""" - Returns `true` if the string can be parsed as a floating-point number. + Returns `true` if the string can be parsed as a 64-bit floating-point number. + """) + boolean isDouble(); + + @Description(""" + Returns `true` if the string can be parsed as a 32-bit floating-point number. """) boolean isFloat(); @Description(""" - Returns `true` if the string can be parsed as an integer. + Returns `true` if the string can be parsed as a 32-bit integer. """) boolean isInteger(); + @Description(""" + Returns `true` if the string can be parsed as a 64-bit integer. + """) + boolean isLong(); + @Description(""" Returns the index within the string of the last occurrence of the given substring. Returns -1 if the string does not contain the substring. """) @@ -146,15 +156,25 @@ public interface String { Boolean toBoolean(); @Description(""" - Parses the string into a floating-point number. + Parses the string into a 64-bit floating-point number. + """) + Double toDouble(); + + @Description(""" + Parses the string into a 32-bit floating-point number. """) Float toFloat(); @Description(""" - Parses the string into an integer. + Parses the string into a 32-bit integer. """) Integer toInteger(); + @Description(""" + Parses the string into a 64-bit integer. + """) + Long toLong(); + @Description(""" Returns a copy of this string with all characters converted to lower case. """) diff --git a/modules/nf-lang/src/test/groovy/nextflow/script/formatter/ScriptFormatterTest.groovy b/modules/nf-lang/src/test/groovy/nextflow/script/formatter/ScriptFormatterTest.groovy index effbd7311e..0ff147b1c9 100644 --- a/modules/nf-lang/src/test/groovy/nextflow/script/formatter/ScriptFormatterTest.groovy +++ b/modules/nf-lang/src/test/groovy/nextflow/script/formatter/ScriptFormatterTest.groovy @@ -37,6 +37,7 @@ class ScriptFormatterTest extends Specification { } String format(String contents) { + scriptParser.compiler().getSources().clear() def source = scriptParser.parse('main.nf', contents) new ScriptResolveVisitor(source, scriptParser.compiler().compilationUnit(), Types.DEFAULT_SCRIPT_IMPORTS, Collections.emptyList()).visit() assert !TestUtils.hasSyntaxErrors(source) @@ -132,6 +133,26 @@ class ScriptFormatterTest extends Specification { } ''' ) + checkFormat( + '''\ + workflow hello{ + take: x:Integer ; y:Integer ; main: xy=x*y ; emit: result:Integer = xy + } + ''', + '''\ + workflow hello { + take: + x: Integer + y: Integer + + main: + xy = x * y + + emit: + result: Integer = xy + } + ''' + ) } def 'should format a process definition' () { @@ -175,6 +196,32 @@ class ScriptFormatterTest extends Specification { } ''' ) + checkFormat( + '''\ + Integer hello(Integer x,Integer y){ + Integer xy=x*y ; return xy + } + ''', + '''\ + def hello(x: Integer, y: Integer) -> Integer { + def xy: Integer = x * y + return xy + } + ''' + ) + checkFormat( + '''\ + def hello(x:Integer,y:Integer)->Integer{ + def xy=x*y ; return xy + } + ''', + '''\ + def hello(x: Integer, y: Integer) -> Integer { + def xy = x * y + return xy + } + ''' + ) } def 'should format an enum definition' () { @@ -216,6 +263,27 @@ class ScriptFormatterTest extends Specification { } ''' ) + checkFormat( + '''\ + output{ + foo:Path{path'foo'} + bar:Channel{path'bar';index{path'index.csv'}} + } + ''', + '''\ + output { + foo: Path { + path 'foo' + } + bar: Channel { + path 'bar' + index { + path 'index.csv' + } + } + } + ''' + ) } def 'should not sort script declarations by default' () { @@ -264,11 +332,29 @@ class ScriptFormatterTest extends Specification { checkFormat( '''\ def x=42 - def(x,y)=[1,2] + def(x,y)=tuple(1,2) ''', '''\ def x = 42 - def (x, y) = [1, 2] + def (x, y) = tuple(1, 2) + ''' + ) + checkFormat( + '''\ + def Integer x=42 + ''', + '''\ + def x: Integer = 42 + ''' + ) + checkFormat( + '''\ + def x:Integer=42 + def(x:Integer,y:Integer)=tuple(1,2) + ''', + '''\ + def x: Integer = 42 + def (x: Integer, y: Integer) = tuple(1, 2) ''' ) } @@ -280,13 +366,13 @@ class ScriptFormatterTest extends Specification { v=42 list[0]='first' map.key='value' - (x,y)=[1,2] + (x,y)=tuple(1,2) ''', '''\ v = 42 list[0] = 'first' map.key = 'value' - (x, y) = [1, 2] + (x, y) = tuple(1, 2) ''' ) } @@ -330,7 +416,7 @@ class ScriptFormatterTest extends Specification { try { println(file('foo.txt').text) } - catch (IOException e) { + catch (e: IOException) { log.warn("Could not load foo.txt") } ''' @@ -484,6 +570,22 @@ class ScriptFormatterTest extends Specification { { a, b -> a + b } ''' ) + checkFormat( + '''\ + {Integer a,Integer b->a+b} + ''', + '''\ + { a: Integer, b: Integer -> a + b } + ''' + ) + checkFormat( + '''\ + {a:Integer,b:Integer->a+b} + ''', + '''\ + { a: Integer, b: Integer -> a + b } + ''' + ) checkFormat( '''\ {v->println'Hello!';v*v} diff --git a/tests/checks/.IGNORE-PARSER-V2 b/tests/checks/.IGNORE-PARSER-V2 index 24e47da582..d90a6e9ec9 100644 --- a/tests/checks/.IGNORE-PARSER-V2 +++ b/tests/checks/.IGNORE-PARSER-V2 @@ -1 +1,2 @@ # TESTS THAT SHOULD ONLY BE RUN BY THE V2 PARSER +type-annotations.nf \ No newline at end of file diff --git a/tests/type-annotations.nf b/tests/type-annotations.nf new file mode 100644 index 0000000000..259a8b252a --- /dev/null +++ b/tests/type-annotations.nf @@ -0,0 +1,40 @@ +#!/usr/bin/env nextflow +/* + * Copyright 2013-2024, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +workflow { + words_ch = channel.of('one', 'two', 'three', 'four') + counts_ch = COUNT(words_ch) + counts_ch.collect().view { counts -> + def even = counts.findAll { n -> isEven(n) }.size() + println "counts: $counts ($even are even)" + } +} + +workflow COUNT { + take: + words: Channel + + main: + counts_ch = words.map { word -> word.length() } + + emit: + counts: Channel = counts_ch +} + +def isEven(n: Integer) -> Boolean { + return n % 2 == 0 +} diff --git a/validation/test.sh b/validation/test.sh index 0096b2984c..c7a40b7857 100755 --- a/validation/test.sh +++ b/validation/test.sh @@ -49,6 +49,7 @@ test_e2e() { # Integration tests # if [[ $TEST_MODE == 'test_integration' ]]; then + export NXF_SYNTAX_PARSER=v1 test_integration ../tests/ test_integration ../tests-v1/ test_e2e