From 99a77413ea4cdb876edf1ed183392d3af6e2f937 Mon Sep 17 00:00:00 2001 From: Masatake YAMATO Date: Sun, 20 Nov 2022 21:08:42 +0900 Subject: [PATCH 1/4] Python: extract reference tags Signed-off-by: Masatake YAMATO --- .../json-output-format.d/stdout-expected.txt | 2 +- Tmain/list-roles.d/stdout-expected.txt | 2 + Units/parser-python.r/reftag.d/args.ctags | 5 ++ Units/parser-python.r/reftag.d/expected.tags | 22 ++++++ Units/parser-python.r/reftag.d/input.py | 18 +++++ parsers/python.c | 75 +++++++++++++++++-- 6 files changed, 115 insertions(+), 9 deletions(-) create mode 100644 Units/parser-python.r/reftag.d/args.ctags create mode 100644 Units/parser-python.r/reftag.d/expected.tags create mode 100644 Units/parser-python.r/reftag.d/input.py diff --git a/Tmain/json-output-format.d/stdout-expected.txt b/Tmain/json-output-format.d/stdout-expected.txt index b8e96b9d5e..021aeef721 100644 --- a/Tmain/json-output-format.d/stdout-expected.txt +++ b/Tmain/json-output-format.d/stdout-expected.txt @@ -75,7 +75,7 @@ {"_type": "ptag", "name": "TAG_KIND_DESCRIPTION", "parserName": "Man", "path": "s,section", "pattern": "sections"} {"_type": "ptag", "name": "TAG_KIND_DESCRIPTION", "parserName": "Man", "path": "t,title", "pattern": "titles"} {"_type": "ptag", "name": "TAG_KIND_DESCRIPTION", "parserName": "Python", "path": "I,namespace", "pattern": "name referring a module defined in other file"} -{"_type": "ptag", "name": "TAG_KIND_DESCRIPTION", "parserName": "Python", "path": "Y,unknown", "pattern": "name referring a class/variable/function/module defined in other module"} +{"_type": "ptag", "name": "TAG_KIND_DESCRIPTION", "parserName": "Python", "path": "Y,unknown", "pattern": "unknwon name"} {"_type": "ptag", "name": "TAG_KIND_DESCRIPTION", "parserName": "Python", "path": "c,class", "pattern": "classes"} {"_type": "ptag", "name": "TAG_KIND_DESCRIPTION", "parserName": "Python", "path": "f,function", "pattern": "functions"} {"_type": "ptag", "name": "TAG_KIND_DESCRIPTION", "parserName": "Python", "path": "i,module", "pattern": "modules"} diff --git a/Tmain/list-roles.d/stdout-expected.txt b/Tmain/list-roles.d/stdout-expected.txt index 32ec05dc78..59b5dc7277 100644 --- a/Tmain/list-roles.d/stdout-expected.txt +++ b/Tmain/list-roles.d/stdout-expected.txt @@ -91,6 +91,7 @@ Protobuf D/protodef imported on imported Protobuf m/message extension on extending the message Python Y/unknown imported on imported from the other module Python Y/unknown indirectlyImported on classes/variables/functions/modules imported in alternative name +Python Y/unknown ref off (EXPERIMENTAL)referenced anyhow Python i/module imported on imported modules Python i/module indirectlyImported on module imported in alternative name Python i/module namespace on namespace from where classes/variables/functions are imported @@ -224,6 +225,7 @@ Protobuf D/protodef imported on imported Protobuf m/message extension on extending the message Python Y/unknown imported on imported from the other module Python Y/unknown indirectlyImported on classes/variables/functions/modules imported in alternative name +Python Y/unknown ref off (EXPERIMENTAL)referenced anyhow Python i/module imported on imported modules Python i/module indirectlyImported on module imported in alternative name Python i/module namespace on namespace from where classes/variables/functions are imported diff --git a/Units/parser-python.r/reftag.d/args.ctags b/Units/parser-python.r/reftag.d/args.ctags new file mode 100644 index 0000000000..6f1b8e0f7d --- /dev/null +++ b/Units/parser-python.r/reftag.d/args.ctags @@ -0,0 +1,5 @@ +--sort=no +--extras=+r +--fields=+r +--kinds-Python=* +--roles-Python.{unknown}=+{ref} diff --git a/Units/parser-python.r/reftag.d/expected.tags b/Units/parser-python.r/reftag.d/expected.tags new file mode 100644 index 0000000000..7a735bfaa4 --- /dev/null +++ b/Units/parser-python.r/reftag.d/expected.tags @@ -0,0 +1,22 @@ +a.b input.py /^import a.b$/;" i roles:imported +object input.py /^class X(object):$/;" Y roles:ref +X input.py /^class X(object):$/;" c roles:def +__init__ input.py /^ def __init__(self, n):$/;" m class:X roles:def +self input.py /^ def __init__(self, n):$/;" z member:X.__init__ file: roles:def +n input.py /^ def __init__(self, n):$/;" z member:X.__init__ file: roles:def +self input.py /^ self.n = n$/;" Y member:X.__init__ roles:ref +n input.py /^ self.n = n$/;" Y member:X.__init__ roles:ref +n input.py /^ self.n = n$/;" Y member:X.__init__ roles:ref +val input.py /^ def val(self):$/;" m class:X roles:def +self input.py /^ def val(self):$/;" z member:X.val file: roles:def +self input.py /^ return self.n$/;" Y member:X.val roles:ref +n input.py /^ return self.n$/;" Y member:X.val roles:ref +one input.py /^def one():$/;" f roles:def +X input.py /^ return X(1)$/;" Y function:one roles:ref +two input.py /^def two():$/;" f roles:def +X input.py /^ return X(2)$/;" Y function:two roles:ref +print input.py /^print (one().val() + two().val())$/;" Y roles:ref +one input.py /^print (one().val() + two().val())$/;" Y roles:ref +val input.py /^print (one().val() + two().val())$/;" Y roles:ref +two input.py /^print (one().val() + two().val())$/;" Y roles:ref +val input.py /^print (one().val() + two().val())$/;" Y roles:ref diff --git a/Units/parser-python.r/reftag.d/input.py b/Units/parser-python.r/reftag.d/input.py new file mode 100644 index 0000000000..0136a53227 --- /dev/null +++ b/Units/parser-python.r/reftag.d/input.py @@ -0,0 +1,18 @@ +import a.b + +class X(object): + def __init__(self, n): + self.n = n + + def val(self): + return self.n + +def one(): + return X(1) + +def two(): + return X(2) + +print (one().val() + two().val()) + + diff --git a/parsers/python.c b/parsers/python.c index e59e60836b..c5fa45a6b5 100644 --- a/parsers/python.c +++ b/parsers/python.c @@ -96,6 +96,7 @@ typedef enum { } pythonModuleRole; typedef enum { + PYTHON_UNKNOWN_REFERENCED, PYTHON_UNKNOWN_IMPORTED, PYTHON_UNKNOWN_INDIRECTLY_IMPORTED, } pythonUnknownRole; @@ -128,6 +129,7 @@ static roleDefinition PythonModuleRoles [] = { }; static roleDefinition PythonUnknownRoles [] = { + { false,"ref", "(EXPERIMENTAL)referenced anyhow" }, { true, "imported", "imported from the other module" }, { true, "indirectlyImported", "classes/variables/functions/modules imported in alternative name" }, @@ -141,7 +143,7 @@ static kindDefinition PythonKinds[COUNT_KIND] = { {true, 'I', "namespace", "name referring a module defined in other file"}, {true, 'i', "module", "modules", .referenceOnly = true, ATTACH_ROLES(PythonModuleRoles)}, - {true, 'Y', "unknown", "name referring a class/variable/function/module defined in other module", + {true, 'Y', "unknown", "unknwon name", .referenceOnly = false, ATTACH_ROLES(PythonUnknownRoles)}, {false, 'z', "parameter", "function parameters" }, {false, 'l', "local", "local variables" }, @@ -201,6 +203,7 @@ typedef struct { int indent; unsigned long lineNumber; MIOPos filePosition; + int reftag; } tokenInfo; struct pythonNestingLevelUserData { @@ -244,13 +247,23 @@ static accessType accessFromIdentifier (const vString *const ident, return ACCESS_PROTECTED; } -static void initPythonEntry (tagEntryInfo *const e, const tokenInfo *const token, +static void useTokenAsPartOfAnotherTag (tokenInfo *const token) +{ + if (token->reftag == CORK_NIL) + return; + + markCorkEntryAsPlaceholder (token->reftag, true); + token->reftag = CORK_NIL; +} + +static void initPythonEntry (tagEntryInfo *const e, tokenInfo *const token, const pythonKind kind) { accessType access; int parentKind = -1; NestingLevel *nl; + useTokenAsPartOfAnotherTag(token); initTagEntry (e, vStringValue (token->string), kind); updateTagLine (e, token->lineNumber, token->filePosition); @@ -283,7 +296,7 @@ static void initPythonEntry (tagEntryInfo *const e, const tokenInfo *const token e->isFileScope = true; } -static int makeClassTag (const tokenInfo *const token, +static int makeClassTag (tokenInfo *const token, const vString *const inheritance, const vString *const decorators) { @@ -306,7 +319,7 @@ static int makeClassTag (const tokenInfo *const token, return CORK_NIL; } -static int makeFunctionTag (const tokenInfo *const token, +static int makeFunctionTag (tokenInfo *const token, const vString *const arglist, const vString *const decorators) { @@ -330,7 +343,7 @@ static int makeFunctionTag (const tokenInfo *const token, return CORK_NIL; } -static int makeSimplePythonTag (const tokenInfo *const token, pythonKind const kind) +static int makeSimplePythonTag (tokenInfo *const token, pythonKind const kind) { if (PythonKinds[kind].enabled) { @@ -343,7 +356,7 @@ static int makeSimplePythonTag (const tokenInfo *const token, pythonKind const k return CORK_NIL; } -static int makeSimplePythonRefTag (const tokenInfo *const token, +static int makeSimplePythonRefTag (tokenInfo *const token, const vString *const altName, pythonKind const kind, int roleIndex, xtagType xtag) @@ -353,6 +366,7 @@ static int makeSimplePythonRefTag (const tokenInfo *const token, { tagEntryInfo e; + useTokenAsPartOfAnotherTag(token); initRefTagEntry (&e, vStringValue (altName ? altName : token->string), kind, roleIndex); @@ -391,6 +405,7 @@ static void clearPoolToken (void *data) token->lineNumber = getInputLineNumber (); token->filePosition = getInputFilePosition (); vStringClear (token->string); + token->reftag = CORK_NIL; } static void copyToken (tokenInfo *const dest, const tokenInfo *const src) @@ -401,6 +416,7 @@ static void copyToken (tokenInfo *const dest, const tokenInfo *const src) dest->keyword = src->keyword; dest->indent = src->indent; vStringCopy(dest->string, src->string); + dest->reftag = src->reftag; } /* Skip a single or double quoted string. */ @@ -477,7 +493,7 @@ static void ungetToken (tokenInfo *const token) copyToken (NextToken, token); } -static void readTokenFull (tokenInfo *const token, bool inclWhitespaces) +static void readTokenFullNoRefTag (tokenInfo *const token, bool inclWhitespaces) { int c; int n; @@ -692,6 +708,36 @@ static void readTokenFull (tokenInfo *const token, bool inclWhitespaces) } } +static void readTokenFull (tokenInfo *const token, bool inclWhitespaces) +{ + readTokenFullNoRefTag (token, inclWhitespaces); + + if (token->type == TOKEN_IDENTIFIER + /* Don't make a ref tag for a number. */ + && (vStringLength (token->string) > 0 && + !isdigit ((unsigned char)vStringChar (token->string, 0))) + && PythonKinds[K_UNKNOWN].enabled + && PythonUnknownRoles[PYTHON_UNKNOWN_REFERENCED].enabled) + { + const bool in_subparser = (Lang_python != getInputLanguage ()); + if (in_subparser) + pushLanguage (Lang_python); + + tagEntryInfo e; + initRefTagEntry(&e, vStringValue (token->string), + K_UNKNOWN, PYTHON_UNKNOWN_REFERENCED); + e.lineNumber = token->lineNumber; + e.filePosition = token->filePosition; + NestingLevel *nl = nestingLevelsGetCurrent (PythonNestingLevels); + if (nl) + e.extensionFields.scopeIndex = nl->corkIndex; + token->reftag = makeTagEntry (&e); + + if (in_subparser) + popLanguage (); + } +} + static void readToken (tokenInfo *const token) { readTokenFull (token, false); @@ -786,6 +832,10 @@ static void readQualifiedName (tokenInfo *const nameToken) vString *qualifiedName = vStringNew (); tokenInfo *token = newToken (); + unsigned long lineNumber = nameToken->lineNumber; + MIOPos filePosition = nameToken->filePosition; + + useTokenAsPartOfAnotherTag (nameToken); while (nameToken->type == TOKEN_IDENTIFIER || nameToken->type == '.') { @@ -793,6 +843,7 @@ static void readQualifiedName (tokenInfo *const nameToken) copyToken (token, nameToken); readToken (nameToken); + useTokenAsPartOfAnotherTag (nameToken); } /* put the last, non-matching, token back */ ungetToken (nameToken); @@ -803,6 +854,13 @@ static void readQualifiedName (tokenInfo *const nameToken) deleteToken (token); vStringDelete (qualifiedName); + + tagEntryInfo e; + initRefTagEntry(&e, vStringValue (nameToken->string), + K_UNKNOWN, PYTHON_UNKNOWN_REFERENCED); + e.lineNumber = lineNumber; + e.filePosition = filePosition; + nameToken->reftag = makeTagEntry (&e); } } @@ -1022,6 +1080,7 @@ static void parseArglist (tokenInfo *const token, const int kind, struct typedParam *parameter; parameterName = newToken (); + useTokenAsPartOfAnotherTag (token); copyToken (parameterName, token); parameterType = parseParamTypeAnnotation (token, arglist); @@ -1488,7 +1547,7 @@ static bool parseVariable (tokenInfo *const token, const pythonKind kind) do { - const tokenInfo *const nameToken = nameTokens[i]; + tokenInfo *const nameToken = nameTokens[i]; vString **type = &(nameTypes[i++]); readToken (token); From cfea32bebd59370b9cef72a4f9e6f022b8c406d8 Mon Sep 17 00:00:00 2001 From: Masatake YAMATO Date: Mon, 5 Dec 2022 21:52:28 +0900 Subject: [PATCH 2/4] Python,refactor: add a function for parsing class inheritance lists --- parsers/python.c | 36 ++++++++++++++++++++++++------------ 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/parsers/python.c b/parsers/python.c index c5fa45a6b5..d51d7ecd15 100644 --- a/parsers/python.c +++ b/parsers/python.c @@ -1039,14 +1039,25 @@ static void deleteTypedParam (struct typedParam *p) eFree (p); } -static void parseArglist (tokenInfo *const token, const int kind, +static void parseInheritanceList (tokenInfo *const token, + vString *const inneritanceList) +{ + do + { + readTokenFull (token, true); + if (token->type != ')') + reprCat (inneritanceList, token); + } + while (token->type != TOKEN_EOF && token->type != ')'); +} + +static void parseArglist (tokenInfo *const token, vString *const arglist, ptrArray *const parameters) { int prevTokenType = token->type; int depth = 1; - if (kind != K_CLASS) - reprCat (arglist, token); + reprCat (arglist, token); do { @@ -1059,8 +1070,7 @@ static void parseArglist (tokenInfo *const token, const int kind, } readTokenFull (token, true); - if (kind != K_CLASS || token->type != ')' || depth > 1) - reprCat (arglist, token); + reprCat (arglist, token); if (token->type == '(' || token->type == '[' || @@ -1070,7 +1080,7 @@ static void parseArglist (tokenInfo *const token, const int kind, token->type == ']' || token->type == '}') depth --; - else if (kind != K_CLASS && depth == 1 && + else if (depth == 1 && token->type == TOKEN_IDENTIFIER && (prevTokenType == '(' || prevTokenType == ',') && PythonKinds[K_PARAMETER].enabled) @@ -1091,8 +1101,8 @@ static void parseArglist (tokenInfo *const token, const int kind, while (token->type != TOKEN_EOF && depth > 0); } -static void parseCArglist (tokenInfo *const token, const int kind, - vString *const arglist, ptrArray *const parameters) +static void parseCArglist (tokenInfo *const token, + vString *const arglist, ptrArray *const parameters) { int depth = 1; tokenInfo *pname = newToken (); @@ -1231,12 +1241,14 @@ static bool parseClassOrDef (tokenInfo *const token, if (token->type == '(') { arglist = vStringNew (); - parameters = ptrArrayNew ((ptrArrayDeleteFunc)deleteTypedParam); + parameters = (kind == K_CLASS)? NULL: ptrArrayNew ((ptrArrayDeleteFunc)deleteTypedParam); - if (isCDef && kind != K_CLASS) - parseCArglist (token, kind, arglist, parameters); + if (kind == K_CLASS) + parseInheritanceList (token, arglist); + else if (isCDef) + parseCArglist (token, arglist, parameters); else - parseArglist (token, kind, arglist, parameters); + parseArglist (token, arglist, parameters); } if (kind == K_CLASS) From 42b349d680442be2dab865a4690a2f91c47254eb Mon Sep 17 00:00:00 2001 From: Masatake YAMATO Date: Mon, 5 Dec 2022 22:26:42 +0900 Subject: [PATCH 3/4] Python: add "super" role to "class" kind Signed-off-by: Masatake YAMATO --- .../json-output-format.d/stdout-expected.txt | 1 + Tmain/list-roles.d/stdout-expected.txt | 2 + Units/parser-python.r/reftag.d/expected.tags | 2 +- .../parser-python.r/simple.py.d/expected.tags | 4 ++ parsers/python.c | 39 ++++++++++++++++++- 5 files changed, 46 insertions(+), 2 deletions(-) diff --git a/Tmain/json-output-format.d/stdout-expected.txt b/Tmain/json-output-format.d/stdout-expected.txt index 021aeef721..250c277e7c 100644 --- a/Tmain/json-output-format.d/stdout-expected.txt +++ b/Tmain/json-output-format.d/stdout-expected.txt @@ -97,6 +97,7 @@ {"_type": "ptag", "name": "TAG_ROLE_DESCRIPTION", "parserName": "C", "kindName": "struct", "path": "foreigndecl", "pattern": "declared in foreign languages"} {"_type": "ptag", "name": "TAG_ROLE_DESCRIPTION", "parserName": "Go", "kindName": "package", "path": "imported", "pattern": "imported package"} {"_type": "ptag", "name": "TAG_ROLE_DESCRIPTION", "parserName": "Go", "kindName": "unknown", "path": "receiverType", "pattern": "receiver type"} +{"_type": "ptag", "name": "TAG_ROLE_DESCRIPTION", "parserName": "Python", "kindName": "class", "path": "super", "pattern": "super class"} {"_type": "ptag", "name": "TAG_ROLE_DESCRIPTION", "parserName": "Python", "kindName": "module", "path": "imported", "pattern": "imported modules"} {"_type": "ptag", "name": "TAG_ROLE_DESCRIPTION", "parserName": "Python", "kindName": "module", "path": "indirectlyImported", "pattern": "module imported in alternative name"} {"_type": "ptag", "name": "TAG_ROLE_DESCRIPTION", "parserName": "Python", "kindName": "module", "path": "namespace", "pattern": "namespace from where classes/variables/functions are imported"} diff --git a/Tmain/list-roles.d/stdout-expected.txt b/Tmain/list-roles.d/stdout-expected.txt index 59b5dc7277..cba7d3b7bd 100644 --- a/Tmain/list-roles.d/stdout-expected.txt +++ b/Tmain/list-roles.d/stdout-expected.txt @@ -92,6 +92,7 @@ Protobuf m/message extension on extending the messag Python Y/unknown imported on imported from the other module Python Y/unknown indirectlyImported on classes/variables/functions/modules imported in alternative name Python Y/unknown ref off (EXPERIMENTAL)referenced anyhow +Python c/class super on super class Python i/module imported on imported modules Python i/module indirectlyImported on module imported in alternative name Python i/module namespace on namespace from where classes/variables/functions are imported @@ -226,6 +227,7 @@ Protobuf m/message extension on extending the messag Python Y/unknown imported on imported from the other module Python Y/unknown indirectlyImported on classes/variables/functions/modules imported in alternative name Python Y/unknown ref off (EXPERIMENTAL)referenced anyhow +Python c/class super on super class Python i/module imported on imported modules Python i/module indirectlyImported on module imported in alternative name Python i/module namespace on namespace from where classes/variables/functions are imported diff --git a/Units/parser-python.r/reftag.d/expected.tags b/Units/parser-python.r/reftag.d/expected.tags index 7a735bfaa4..8f713c2f5d 100644 --- a/Units/parser-python.r/reftag.d/expected.tags +++ b/Units/parser-python.r/reftag.d/expected.tags @@ -1,5 +1,5 @@ a.b input.py /^import a.b$/;" i roles:imported -object input.py /^class X(object):$/;" Y roles:ref +object input.py /^class X(object):$/;" c roles:super X input.py /^class X(object):$/;" c roles:def __init__ input.py /^ def __init__(self, n):$/;" m class:X roles:def self input.py /^ def __init__(self, n):$/;" z member:X.__init__ file: roles:def diff --git a/Units/parser-python.r/simple.py.d/expected.tags b/Units/parser-python.r/simple.py.d/expected.tags index 6379eef842..a7891e5344 100644 --- a/Units/parser-python.r/simple.py.d/expected.tags +++ b/Units/parser-python.r/simple.py.d/expected.tags @@ -37,9 +37,13 @@ this input.py /^ @blah class this is seen???$/;" kind:class l _test.ignored_function.more_nesting.deeply_nested.even_more.this input.py /^ @blah class this is seen???$/;" kind:class line:41 language:Python scope:member:_test.ignored_function.more_nesting.deeply_nested.even_more file: inherits: access:private roles:def extras:qualified end:42 decorators:blah this input.py /^ @bleh def this also? good!$/;" kind:member line:42 language:Python scope:class:_test.ignored_function.more_nesting.deeply_nested.even_more.this access:public roles:def end:42 decorators:bleh _test.ignored_function.more_nesting.deeply_nested.even_more.this.this input.py /^ @bleh def this also? good!$/;" kind:member line:42 language:Python scope:class:_test.ignored_function.more_nesting.deeply_nested.even_more.this access:public roles:def extras:qualified end:42 decorators:bleh +one input.py /^class two (one):$/;" kind:class line:46 language:Python roles:super extras:reference two input.py /^class two (one):$/;" kind:class line:46 language:Python inherits:one access:public roles:def end:48 only input.py /^ def only(arg):$/;" kind:member line:48 language:Python scope:class:two access:public signature:(arg) roles:def end:48 two.only input.py /^ def only(arg):$/;" kind:member line:48 language:Python scope:class:two access:public signature:(arg) roles:def extras:qualified end:48 +A input.py /^(A, B,$/;" kind:class line:53 language:Python roles:super extras:reference +B input.py /^(A, B,$/;" kind:class line:53 language:Python roles:super extras:reference +C input.py /^C):$/;" kind:class line:54 language:Python roles:super extras:reference three input.py /^three\\$/;" kind:class line:52 language:Python inherits:A, B, C access:public roles:def end:54 foo input.py /^foo($/;" kind:function line:57 language:Python access:public signature:( x , y, z) roles:def end:60 input.py input.py 1;" kind:file line:1 language:Python roles:def extras:inputFile end:60 diff --git a/parsers/python.c b/parsers/python.c index d51d7ecd15..28d9be2d8d 100644 --- a/parsers/python.c +++ b/parsers/python.c @@ -101,6 +101,10 @@ typedef enum { PYTHON_UNKNOWN_INDIRECTLY_IMPORTED, } pythonUnknownRole; +typedef enum { + PYTHON_CLASS_SUPERCLASS, +} pythonClassRole; + /* Roles related to `import' * ========================== * import X X = (kind:module, role:imported) @@ -135,8 +139,13 @@ static roleDefinition PythonUnknownRoles [] = { "classes/variables/functions/modules imported in alternative name" }, }; +static roleDefinition PythonClassRoles [] = { + { true, "super", "super class" }, +}; + static kindDefinition PythonKinds[COUNT_KIND] = { - {true, 'c', "class", "classes"}, + {true, 'c', "class", "classes", + .referenceOnly = false, ATTACH_ROLES(PythonClassRoles) }, {true, 'f', "function", "functions"}, {true, 'm', "member", "class members"}, {true, 'v', "variable", "variables"}, @@ -1042,13 +1051,41 @@ static void deleteTypedParam (struct typedParam *p) static void parseInheritanceList (tokenInfo *const token, vString *const inneritanceList) { + tokenInfo *lastToken = newToken (); + do { + copyToken (lastToken, token); readTokenFull (token, true); + if (lastToken->type == TOKEN_IDENTIFIER + && (token->type == ',' || token->type == ')')) + { + if (lastToken->reftag == CORK_NIL) + { + tagEntryInfo e; + initRefTagEntry(&e, vStringValue (lastToken->string), + K_CLASS, PYTHON_CLASS_SUPERCLASS); + e.lineNumber = lastToken->lineNumber; + e.filePosition = lastToken->filePosition; + lastToken->reftag = makeTagEntry (&e); + } + else + { + tagEntryInfo *e = getEntryInCorkQueue (lastToken->reftag); + if (e) + { + clearRoles(e); + e->kindIndex = K_CLASS; + assignRole(e, PYTHON_CLASS_SUPERCLASS); + } + } + } if (token->type != ')') reprCat (inneritanceList, token); } while (token->type != TOKEN_EOF && token->type != ')'); + + deleteToken (lastToken); } static void parseArglist (tokenInfo *const token, From 3ffd74be86d5f2c6f730fb7e36aae9fceda1e3dd Mon Sep 17 00:00:00 2001 From: Masatake YAMATO Date: Tue, 19 Dec 2023 05:47:54 +0900 Subject: [PATCH 4/4] Python: don't make reference tags for 'self' Signed-off-by: Masatake YAMATO --- Units/parser-python.r/reftag.d/expected.tags | 2 -- parsers/python.c | 18 ++++++++++++++++-- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/Units/parser-python.r/reftag.d/expected.tags b/Units/parser-python.r/reftag.d/expected.tags index 8f713c2f5d..8cd3e61fc0 100644 --- a/Units/parser-python.r/reftag.d/expected.tags +++ b/Units/parser-python.r/reftag.d/expected.tags @@ -4,12 +4,10 @@ X input.py /^class X(object):$/;" c roles:def __init__ input.py /^ def __init__(self, n):$/;" m class:X roles:def self input.py /^ def __init__(self, n):$/;" z member:X.__init__ file: roles:def n input.py /^ def __init__(self, n):$/;" z member:X.__init__ file: roles:def -self input.py /^ self.n = n$/;" Y member:X.__init__ roles:ref n input.py /^ self.n = n$/;" Y member:X.__init__ roles:ref n input.py /^ self.n = n$/;" Y member:X.__init__ roles:ref val input.py /^ def val(self):$/;" m class:X roles:def self input.py /^ def val(self):$/;" z member:X.val file: roles:def -self input.py /^ return self.n$/;" Y member:X.val roles:ref n input.py /^ return self.n$/;" Y member:X.val roles:ref one input.py /^def one():$/;" f roles:def X input.py /^ return X(1)$/;" Y function:one roles:ref diff --git a/parsers/python.c b/parsers/python.c index 28d9be2d8d..693a6b1cbd 100644 --- a/parsers/python.c +++ b/parsers/python.c @@ -44,6 +44,11 @@ enum { KEYWORD_lambda, KEYWORD_pass, KEYWORD_return, + + /* Used only in readTokenFullNoRefTag to represent the identifiers + that should not be tagged as reference tags. */ + KEYWORD___noreftag_id, + KEYWORD_REST }; typedef int keywordId; /* to allow KEYWORD_NONE */ @@ -174,6 +179,7 @@ static const keywordTable PythonKeywordTable[] = { { "lambda", KEYWORD_lambda }, { "pass", KEYWORD_pass }, { "return", KEYWORD_return }, + { "self", KEYWORD___noreftag_id }, }; /* Taken from https://docs.python.org/3/reference/lexical_analysis.html#keywords */ @@ -502,7 +508,7 @@ static void ungetToken (tokenInfo *const token) copyToken (NextToken, token); } -static void readTokenFullNoRefTag (tokenInfo *const token, bool inclWhitespaces) +static void readTokenFullNoRefTag (tokenInfo *const token, bool inclWhitespaces, bool *noReftagId) { int c; int n; @@ -689,11 +695,17 @@ static void readTokenFullNoRefTag (tokenInfo *const token, bool inclWhitespaces) } else { + *noReftagId = false; /* FIXME: handle U, B, R and F string prefixes? */ readIdentifier (token->string, c); token->keyword = lookupKeyword (vStringValue (token->string), Lang_python); if (token->keyword == KEYWORD_NONE) token->type = TOKEN_IDENTIFIER; + else if (token->keyword == KEYWORD___noreftag_id) + { + token->type = TOKEN_IDENTIFIER; + *noReftagId = true; + } else token->type = TOKEN_KEYWORD; } @@ -719,9 +731,11 @@ static void readTokenFullNoRefTag (tokenInfo *const token, bool inclWhitespaces) static void readTokenFull (tokenInfo *const token, bool inclWhitespaces) { - readTokenFullNoRefTag (token, inclWhitespaces); + bool noReftagId; + readTokenFullNoRefTag (token, inclWhitespaces, &noReftagId); if (token->type == TOKEN_IDENTIFIER + && (!noReftagId) /* Don't make a ref tag for a number. */ && (vStringLength (token->string) > 0 && !isdigit ((unsigned char)vStringChar (token->string, 0)))