utokenize: Add module from pycopy v2.0.

source: https://github.com/pfalcon/pycopy-lib/tree/6a5a47b82b84deed49f6f83c22e44620286f8100/utokenize

Author: pi-anl

python-stdlib/utokenize/example_dump.py

@@ -0,0 +1,7 @@
+import sys
+import utokenize as tokenize
+
+
+f = open(sys.argv[1], "r")
+for t in tokenize.generate_tokens(f.readline):
+    print(t)

python-stdlib/utokenize/example_test_dump.py

@@ -0,0 +1,11 @@
+import sys
+import utokenize as tokenize
+
+
+f = open(sys.argv[1], "r")
+for t in tokenize.generate_tokens(f.readline):
+    # print(t)
+    print(
+        "TokenInfo(type=%d (%s), string=%r, startl=%d)"
+        % (t.type, tokenize.tok_name[t.type], t.string, t.start)
+    )

python-stdlib/utokenize/example_test_dump_cpy.py

@@ -0,0 +1,12 @@
+# Same as example_dump.py, but for CPython's tokenize module
+import sys
+import tokenize as tokenize
+
+
+f = open(sys.argv[1], "r")
+for t in tokenize.generate_tokens(f.readline):
+    # print(t)
+    print(
+        "TokenInfo(type=%d (%s), string=%r, startl=%d)"
+        % (t.type, tokenize.tok_name[t.type], t.string, t.start[0])
+    )

python-stdlib/utokenize/metadata.txt

@@ -0,0 +1,5 @@
+srctype = micropython-lib
+type = module
+version = 2.0
+author = Paul Sokolovsky
+long_desc = Simple tokenizer for Python source code.

python-stdlib/utokenize/test_corpus.py

@@ -0,0 +1,32 @@
+import sys
+import os
+import glob
+
+
+def handle_one(f):
+    rc = os.system("pycopy example_test_dump.py %s >%s.upy" % (f, f))
+    if rc != 0:
+        print("%s: FAIL" % f)
+        return False
+    rc = os.system("diff -u %s.cpy %s.upy" % (f, f))
+    if rc != 0:
+        print("%s: error" % f)
+        return False
+    print("%s: ok" % f)
+    return True
+
+
+if len(sys.argv) > 1 and sys.argv[1] == "--make-expected":
+    for f in glob.glob("testdata/*.py"):
+        rc = os.system("python3.6 example_test_dump_cpy.py %s >%s.cpy" % (f, f))
+        assert rc == 0
+    sys.exit()
+
+
+errors = False
+
+for f in sorted(glob.glob("testdata/*.py")):
+    e = handle_one(f)
+    errors = errors or not e
+
+sys.exit(int(errors))

python-stdlib/utokenize/testdata/.gitignore

@@ -0,0 +1,2 @@
+*.cpy
+*.upy

python-stdlib/utokenize/testdata/basic.py

@@ -0,0 +1,5 @@
+def foo():
+    print(1)
+    print(2)
+
+foo()

python-stdlib/utokenize/testdata/basic_empty_lines.py

@@ -0,0 +1,14 @@
+def foo():
+    print(1)
+
+    print(2)
+
+
+    print(3)
+ 
+    print(4)
+                                 
+    print(5)
+
+
+foo()

python-stdlib/utokenize/testdata/basic_multilevel.py

@@ -0,0 +1,4 @@
+def foo():
+    if 1:
+        if 2:
+            pass

python-stdlib/utokenize/testdata/comment_intra.py

@@ -0,0 +1 @@
+foo # comment