From 283ab92fe35f12632a33e3ed33e534d2b5b92ef5 Mon Sep 17 00:00:00 2001 From: Lukasz Konopski Date: Wed, 31 Aug 2016 16:03:59 +0200 Subject: [PATCH 01/15] skip standard java build artifacts --- grin.py | 8 ++++---- setup.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/grin.py b/grin.py index bf42dc0..edf0784 100755 --- a/grin.py +++ b/grin.py @@ -816,13 +816,13 @@ def get_grin_arg_parser(parser=None): default=True, action='store_true', help="do skip .hidden directories [default]") parser.add_argument('-d', '--skip-dirs', - default='CVS,RCS,.svn,.hg,.bzr,build,dist', + default='CVS,RCS,.svn,.hg,.bzr,build,dist,target', help="comma-separated list of directory names to skip [default=%(default)r]") parser.add_argument('-D', '--no-skip-dirs', dest='skip_dirs', action='store_const', const='', help="do not skip any directories") parser.add_argument('-e', '--skip-exts', - default='.pyc,.pyo,.so,.o,.a,.tgz,.tar.gz,.rar,.zip,~,#,.bak,.png,.jpg,.gif,.bmp,.tif,.tiff,.pyd,.dll,.exe,.obj,.lib', + default='.pyc,.pyo,.so,.o,.a,.tgz,.tar.gz,.rar,.zip,~,#,.bak,.png,.jpg,.gif,.bmp,.tif,.tiff,.pyd,.dll,.exe,.obj,.lib,.class', help="comma-separated list of file extensions to skip [default=%(default)r]") parser.add_argument('-E', '--no-skip-exts', dest='skip_exts', action='store_const', const='', @@ -874,13 +874,13 @@ def get_grind_arg_parser(parser=None): default=True, action='store_true', help="do skip .hidden directories") parser.add_argument('-d', '--skip-dirs', - default='CVS,RCS,.svn,.hg,.bzr,build,dist', + default='CVS,RCS,.svn,.hg,.bzr,build,dist,target', help="comma-separated list of directory names to skip [default=%(default)r]") parser.add_argument('-D', '--no-skip-dirs', dest='skip_dirs', action='store_const', const='', help="do not skip any directories") parser.add_argument('-e', '--skip-exts', - default='.pyc,.pyo,.so,.o,.a,.tgz,.tar.gz,.rar,.zip,~,#,.bak,.png,.jpg,.gif,.bmp,.tif,.tiff,.pyd,.dll,.exe,.obj,.lib', + default='.pyc,.pyo,.so,.o,.a,.tgz,.tar.gz,.rar,.zip,~,#,.bak,.png,.jpg,.gif,.bmp,.tif,.tiff,.pyd,.dll,.exe,.obj,.lib,.class', help="comma-separated list of file extensions to skip [default=%(default)r]") parser.add_argument('-E', '--no-skip-exts', dest='skip_exts', action='store_const', const='', diff --git a/setup.py b/setup.py index 3d1f1eb..e017e52 100644 --- a/setup.py +++ b/setup.py @@ -12,7 +12,7 @@ setup( name='grin', - version='1.2.1', + version='1.2.2', author='Robert Kern', author_email='robert.kern@enthought.com', description="A grep program configured the way I like it.", From d03df7a514cd9eecfc26fa831b42f563885f2655 Mon Sep 17 00:00:00 2001 From: Matthew Brett Date: Tue, 6 Sep 2016 13:00:05 -0700 Subject: [PATCH 02/15] MAINT: Python 3 compatibility and travis testing Use ``futurize`` and some hand editing to update code for Python3. Add .travis.yml file for travis-ci testing. --- .gitignore | 2 + .travis.yml | 14 +++++ examples/grinimports.py | 5 +- examples/grinpython.py | 4 +- grin.py | 47 ++++++++------ tests/test_file_recognizer.py | 47 +++++++------- tests/test_grep.py | 112 +++++++++++++++++----------------- 7 files changed, 126 insertions(+), 105 deletions(-) create mode 100644 .travis.yml diff --git a/.gitignore b/.gitignore index ac42661..e8a4072 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ build/ dist/ grin.egg-info/ +__pycache__/ +*.pyc diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..a2d9eb2 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,14 @@ +language: python + +python: + - 2.6 + - 2.7 + - 3.3 + - 3.4 + - 3.5 + +install: + - pip install -e . + +script: + - nosetests . diff --git a/examples/grinimports.py b/examples/grinimports.py index b6d48ff..fa4917f 100755 --- a/examples/grinimports.py +++ b/examples/grinimports.py @@ -2,10 +2,9 @@ # -*- coding: UTF-8 -*- """ Transform Python files into normalized import statements for grepping. """ - import compiler from compiler.visitor import ASTVisitor, walk -from cStringIO import StringIO +from io import StringIO import os import shlex import sys @@ -70,7 +69,7 @@ def normalize_file(filename, *args): """ try: ast = compiler.parseFile(filename) - except Exception, e: + except Exception as e: return StringIO('') ip = ImportPuller() walk(ast, ip) diff --git a/examples/grinpython.py b/examples/grinpython.py index 31e23ee..a9e1395 100755 --- a/examples/grinpython.py +++ b/examples/grinpython.py @@ -3,7 +3,7 @@ """ Transform Python code by omitting strings, comments, and/or code. """ -from cStringIO import StringIO +from io import BytesIO import os import shlex import string @@ -55,7 +55,7 @@ def __call__(self, filename, mode='rb'): """ Open a file and convert it to a filelike object with transformed contents. """ - g = StringIO() + g = BytesIO() f = open(filename, mode) try: gen = tokenize.generate_tokens(f.readline) diff --git a/grin.py b/grin.py index bf42dc0..aa33bb2 100755 --- a/grin.py +++ b/grin.py @@ -1,6 +1,7 @@ #!/usr/bin/env python """ grin searches text files. """ +from __future__ import print_function import bisect import fnmatch @@ -11,9 +12,16 @@ import shlex import stat import sys +from io import UnsupportedOperation import argparse +if sys.version_info[0] > 2: + to_str = lambda s : s.decode('latin1') + ints2bytes = bytes +else: + to_str = str + ints2bytes = lambda ints : ''.join(map(chr, ints)) #### Constants #### __version__ = '1.2.1' @@ -24,8 +32,8 @@ POST = 1 # Use file(1)'s choices for what's text and what's not. -TEXTCHARS = ''.join(map(chr, [7,8,9,10,12,13,27] + range(0x20, 0x100))) -ALLBYTES = ''.join(map(chr, range(256))) +TEXTCHARS = ints2bytes([7,8,9,10,12,13,27] + list(range(0x20, 0x100))) +ALLBYTES = ints2bytes(range(256)) COLOR_TABLE = ['black', 'red', 'green', 'yellow', 'blue', 'magenta', 'cyan', 'white', 'default'] @@ -35,12 +43,11 @@ } # gzip magic header bytes. -GZIP_MAGIC = '\037\213' +GZIP_MAGIC = b'\037\213' # Target amount of data to read into memory at a time. READ_BLOCKSIZE = 16 * 1024 * 1024 - def is_binary_string(bytes): """ Determine if a string is classified as binary rather than text. @@ -54,7 +61,8 @@ def is_binary_string(bytes): """ nontext = bytes.translate(ALLBYTES, TEXTCHARS) return bool(nontext) - + + def get_line_offsets(block): """ Compute the list of offsets in DataBlock 'block' which correspond to the beginnings of new lines. @@ -80,7 +88,8 @@ def get_line_offsets(block): # Keep track of the count of lines within the "current block" if next_newline >= block.start and next_newline < block.end: line_count += 1 - + + def colorize(s, fg=None, bg=None, bold=False, underline=False, reverse=False): """ Wraps a string with ANSI color escape sequences corresponding to the style parameters given. @@ -207,7 +216,7 @@ def __init__(self, regex, options=None): def read_block_with_context(self, prev, fp, fp_size): """ Read a block of data from the file, along with some surrounding context. - + Parameters ---------- prev : DataBlock, or None @@ -216,11 +225,11 @@ def read_block_with_context(self, prev, fp, fp_size): fp : filelike object The source of block data. - + fp_size : int or None Size of the file in bytes, or None if the size could not be determined. - + Returns ------- A DataBlock representing the "current" block along with context. @@ -241,7 +250,7 @@ def read_block_with_context(self, prev, fp, fp_size): # can avoid the overhead of locating lines of 'before' and # 'after' context. result = DataBlock( - data = block_main, + data = to_str(block_main), start = 0, end = len(block_main), before_count = 0, @@ -280,7 +289,7 @@ def read_block_with_context(self, prev, fp, fp_size): after_lines = ''.join(after_lines_list) result = DataBlock( - data = before_lines + curr_block + after_lines, + data = to_str(before_lines + curr_block + after_lines), start = len(before_lines), end = len(before_lines) + len(curr_block), before_count = before_count, @@ -308,13 +317,15 @@ def do_grep(self, fp): fp_size = None # gzipped data is usually longer than the file else: try: - status = os.fstat(fp.fileno()) + file_no = fp.fileno() + except (AttributeError, UnsupportedOperation): # doesn't support fileno() + fp_size = None + else: + status = os.fstat(file_no) if stat.S_ISREG(status.st_mode): fp_size = status.st_size else: fp_size = None - except AttributeError: # doesn't support fileno() - fp_size = None block = self.read_block_with_context(None, fp, fp_size) while block.end > block.start: @@ -587,7 +598,7 @@ def _is_binary_file(self, f): """ try: bytes = f.read(self.binary_bytes) - except Exception, e: + except Exception as e: # When trying to read from something that looks like a gzipped file, # it may be corrupt. If we do get an error, assume that the file is binary. return True @@ -1032,7 +1043,7 @@ def grin_main(argv=None): sys.stdout.write(report) except KeyboardInterrupt: raise SystemExit(0) - except IOError, e: + except IOError as e: if 'Broken pipe' in str(e): # The user is probably piping to a pager like less(1) and has exited # it. Just exit. @@ -1040,7 +1051,7 @@ def grin_main(argv=None): raise def print_line(filename): - print filename + print(filename) def print_null(filename): # Note that the final filename will have a trailing NUL, just like @@ -1073,7 +1084,7 @@ def grind_main(argv=None): output(filename) except KeyboardInterrupt: raise SystemExit(0) - except IOError, e: + except IOError as e: if 'Broken pipe' in str(e): # The user is probably piping to a pager like less(1) and has exited # it. Just exit. diff --git a/tests/test_file_recognizer.py b/tests/test_file_recognizer.py index 70b7f5a..a085ce1 100644 --- a/tests/test_file_recognizer.py +++ b/tests/test_file_recognizer.py @@ -1,5 +1,6 @@ """ Test the file recognizer capabilities. """ +from __future__ import print_function import gzip import os @@ -9,7 +10,7 @@ import nose -from grin import FileRecognizer +from grin import FileRecognizer, ints2bytes, GZIP_MAGIC def empty_file(filename, open=open): f = open(filename, 'wb') @@ -17,13 +18,13 @@ def empty_file(filename, open=open): def binary_file(filename, open=open): f = open(filename, 'wb') - f.write(''.join(map(chr, range(256)))) + f.write(ints2bytes(range(255))) f.close() def text_file(filename, open=open): - lines = ['foo\n', 'bar\n'] * 100 - lines.append('baz\n') - lines.extend(['foo\n', 'bar\n'] * 100) + lines = [b'foo\n', b'bar\n'] * 100 + lines.append(b'baz\n') + lines.extend([b'foo\n', b'bar\n'] * 100) f = open(filename, 'wb') f.writelines(lines) f.close() @@ -32,10 +33,9 @@ def fake_gzip_file(filename, open=open): """ Write out a binary file that has the gzip magic header bytes, but is not a gzip file. """ - GZIP_MAGIC = '\037\213' f = open(filename, 'wb') f.write(GZIP_MAGIC) - f.write(''.join(map(chr, range(256)))) + f.write(ints2bytes(range(255))) f.close() def binary_middle(filename, open=open): @@ -43,7 +43,7 @@ def binary_middle(filename, open=open): bytes, then 100 text bytes to test that the recognizer only reads some of the file. """ - text = 'a'*100 + '\0'*100 + 'b'*100 + text = b'a'*100 + b'\0'*100 + b'b'*100 f = open(filename, 'wb') f.write(text) f.close() @@ -56,25 +56,25 @@ def unreadable_file(filename): """ Write a file that does not have read permissions. """ text_file(filename) - os.chmod(filename, 0200) + os.chmod(filename, 0o200) def unreadable_dir(filename): """ Make a directory that does not have read permissions. """ os.mkdir(filename) - os.chmod(filename, 0300) + os.chmod(filename, 0o300) def unexecutable_dir(filename): """ Make a directory that does not have execute permissions. """ os.mkdir(filename) - os.chmod(filename, 0600) + os.chmod(filename, 0o600) def totally_unusable_dir(filename): """ Make a directory that has neither read nor execute permissions. """ os.mkdir(filename) - os.chmod(filename, 0100) + os.chmod(filename, 0o100) def setup(): # Make files to test individual recognizers. @@ -135,22 +135,14 @@ def setup(): text_file('tree/.skip_hidden_file') os.mkdir('tree/unreadable_dir') text_file('tree/unreadable_dir/text') - os.chmod('tree/unreadable_dir', 0300) + os.chmod('tree/unreadable_dir', 0o300) os.mkdir('tree/unexecutable_dir') text_file('tree/unexecutable_dir/text') - os.chmod('tree/unexecutable_dir', 0600) + os.chmod('tree/unexecutable_dir', 0o600) os.mkdir('tree/totally_unusable_dir') text_file('tree/totally_unusable_dir/text') - os.chmod('tree/totally_unusable_dir', 0100) + os.chmod('tree/totally_unusable_dir', 0o100) -def ensure_deletability(arg, dirname, fnames): - """ os.path.walk() callback function which will make sure every directory is - readable and executable so that it may be easily deleted. - """ - for fn in fnames: - fn = os.path.join(dirname, fn) - if os.path.isdir(fn): - os.chmod(fn, 0700) def teardown(): files_to_delete = ['empty', 'binary', 'binary_middle', 'text', 'text~', @@ -168,10 +160,13 @@ def teardown(): os.unlink(filename) else: os.rmdir(filename) - except Exception, e: - print >>sys.stderr, 'Could not delete %s: %s' % (filename, e) + except Exception as e: + print('Could not delete %s: %s' % (filename, e), file=sys.stderr) os.unlink('socket_test') - os.path.walk('tree', ensure_deletability, None) + for dirpath, dirnames, filenames in os.walk('tree'): + # Make sure every directory can be deleted + for dirname in dirnames: + os.chmod(os.path.join(dirpath, dirname), 0o700) shutil.rmtree('tree') diff --git a/tests/test_grep.py b/tests/test_grep.py index aa367f2..f5ee62f 100644 --- a/tests/test_grep.py +++ b/tests/test_grep.py @@ -4,52 +4,52 @@ Set up >>> import grin - >>> from cStringIO import StringIO + >>> from io import BytesIO >>> import re - >>> - >>> all_foo = """\ + >>> + >>> all_foo = b"""\ ... foo ... foo ... foo ... foo ... foo ... """ - >>> first_foo = """\ + >>> first_foo = b"""\ ... foo ... bar ... bar ... bar ... bar ... """ - >>> last_foo = """\ + >>> last_foo = b"""\ ... bar ... bar ... bar ... bar ... foo ... """ - >>> second_foo = """\ + >>> second_foo = b"""\ ... bar ... foo ... bar ... bar ... bar ... """ - >>> second_last_foo = """\ + >>> second_last_foo = b"""\ ... bar ... bar ... bar ... foo ... bar ... """ - >>> middle_foo = """\ + >>> middle_foo = b"""\ ... bar ... bar ... foo ... bar ... bar ... """ - >>> small_gap = """\ + >>> small_gap = b"""\ ... bar ... bar ... foo @@ -58,8 +58,8 @@ ... bar ... bar ... """ - >>> no_eol = "foo" - >>> middle_of_line = """\ + >>> no_eol = b"foo" + >>> middle_of_line = b"""\ ... bar ... bar ... barfoobar @@ -70,111 +70,111 @@ Test the basic defaults, no context. >>> gt_default = grin.GrepText(re.compile('foo')) - >>> gt_default.do_grep(StringIO(all_foo)) + >>> gt_default.do_grep(BytesIO(all_foo)) [(0, 0, 'foo\n', [(0, 3)]), (1, 0, 'foo\n', [(0, 3)]), (2, 0, 'foo\n', [(0, 3)]), (3, 0, 'foo\n', [(0, 3)]), (4, 0, 'foo\n', [(0, 3)])] - >>> gt_default.do_grep(StringIO(first_foo)) + >>> gt_default.do_grep(BytesIO(first_foo)) [(0, 0, 'foo\n', [(0, 3)])] - >>> gt_default.do_grep(StringIO(last_foo)) + >>> gt_default.do_grep(BytesIO(last_foo)) [(4, 0, 'foo\n', [(0, 3)])] - >>> gt_default.do_grep(StringIO(second_foo)) + >>> gt_default.do_grep(BytesIO(second_foo)) [(1, 0, 'foo\n', [(0, 3)])] - >>> gt_default.do_grep(StringIO(second_last_foo)) + >>> gt_default.do_grep(BytesIO(second_last_foo)) [(3, 0, 'foo\n', [(0, 3)])] - >>> gt_default.do_grep(StringIO(middle_foo)) + >>> gt_default.do_grep(BytesIO(middle_foo)) [(2, 0, 'foo\n', [(0, 3)])] - >>> gt_default.do_grep(StringIO(small_gap)) + >>> gt_default.do_grep(BytesIO(small_gap)) [(2, 0, 'foo\n', [(0, 3)]), (4, 0, 'foo\n', [(0, 3)])] - >>> gt_default.do_grep(StringIO(no_eol)) + >>> gt_default.do_grep(BytesIO(no_eol)) [(0, 0, 'foo', [(0, 3)])] - >>> gt_default.do_grep(StringIO(middle_of_line)) + >>> gt_default.do_grep(BytesIO(middle_of_line)) [(2, 0, 'barfoobar\n', [(3, 6)])] Symmetric 1-line context. >>> gt_context_1 = grin.GrepText(re.compile('foo'), options=grin.Options(before_context=1, after_context=1)) - >>> gt_context_1.do_grep(StringIO(all_foo)) + >>> gt_context_1.do_grep(BytesIO(all_foo)) [(0, 0, 'foo\n', [(0, 3)]), (1, 0, 'foo\n', [(0, 3)]), (2, 0, 'foo\n', [(0, 3)]), (3, 0, 'foo\n', [(0, 3)]), (4, 0, 'foo\n', [(0, 3)])] - >>> gt_context_1.do_grep(StringIO(first_foo)) + >>> gt_context_1.do_grep(BytesIO(first_foo)) [(0, 0, 'foo\n', [(0, 3)]), (1, 1, 'bar\n', None)] - >>> gt_context_1.do_grep(StringIO(last_foo)) + >>> gt_context_1.do_grep(BytesIO(last_foo)) [(3, -1, 'bar\n', None), (4, 0, 'foo\n', [(0, 3)])] - >>> gt_context_1.do_grep(StringIO(second_foo)) + >>> gt_context_1.do_grep(BytesIO(second_foo)) [(0, -1, 'bar\n', None), (1, 0, 'foo\n', [(0, 3)]), (2, 1, 'bar\n', None)] - >>> gt_context_1.do_grep(StringIO(second_last_foo)) + >>> gt_context_1.do_grep(BytesIO(second_last_foo)) [(2, -1, 'bar\n', None), (3, 0, 'foo\n', [(0, 3)]), (4, 1, 'bar\n', None)] - >>> gt_context_1.do_grep(StringIO(middle_foo)) + >>> gt_context_1.do_grep(BytesIO(middle_foo)) [(1, -1, 'bar\n', None), (2, 0, 'foo\n', [(0, 3)]), (3, 1, 'bar\n', None)] - >>> gt_context_1.do_grep(StringIO(small_gap)) + >>> gt_context_1.do_grep(BytesIO(small_gap)) [(1, -1, 'bar\n', None), (2, 0, 'foo\n', [(0, 3)]), (3, 1, 'bar\n', None), (4, 0, 'foo\n', [(0, 3)]), (5, 1, 'bar\n', None)] - >>> gt_context_1.do_grep(StringIO(no_eol)) + >>> gt_context_1.do_grep(BytesIO(no_eol)) [(0, 0, 'foo', [(0, 3)])] - >>> gt_context_1.do_grep(StringIO(middle_of_line)) + >>> gt_context_1.do_grep(BytesIO(middle_of_line)) [(1, -1, 'bar\n', None), (2, 0, 'barfoobar\n', [(3, 6)]), (3, 1, 'bar\n', None)] Symmetric 2-line context. >>> gt_context_2 = grin.GrepText(re.compile('foo'), options=grin.Options(before_context=2, after_context=2)) - >>> gt_context_2.do_grep(StringIO(all_foo)) + >>> gt_context_2.do_grep(BytesIO(all_foo)) [(0, 0, 'foo\n', [(0, 3)]), (1, 0, 'foo\n', [(0, 3)]), (2, 0, 'foo\n', [(0, 3)]), (3, 0, 'foo\n', [(0, 3)]), (4, 0, 'foo\n', [(0, 3)])] - >>> gt_context_2.do_grep(StringIO(first_foo)) + >>> gt_context_2.do_grep(BytesIO(first_foo)) [(0, 0, 'foo\n', [(0, 3)]), (1, 1, 'bar\n', None), (2, 1, 'bar\n', None)] - >>> gt_context_2.do_grep(StringIO(last_foo)) + >>> gt_context_2.do_grep(BytesIO(last_foo)) [(2, -1, 'bar\n', None), (3, -1, 'bar\n', None), (4, 0, 'foo\n', [(0, 3)])] - >>> gt_context_2.do_grep(StringIO(second_foo)) + >>> gt_context_2.do_grep(BytesIO(second_foo)) [(0, -1, 'bar\n', None), (1, 0, 'foo\n', [(0, 3)]), (2, 1, 'bar\n', None), (3, 1, 'bar\n', None)] - >>> gt_context_2.do_grep(StringIO(second_last_foo)) + >>> gt_context_2.do_grep(BytesIO(second_last_foo)) [(1, -1, 'bar\n', None), (2, -1, 'bar\n', None), (3, 0, 'foo\n', [(0, 3)]), (4, 1, 'bar\n', None)] - >>> gt_context_2.do_grep(StringIO(middle_foo)) + >>> gt_context_2.do_grep(BytesIO(middle_foo)) [(0, -1, 'bar\n', None), (1, -1, 'bar\n', None), (2, 0, 'foo\n', [(0, 3)]), (3, 1, 'bar\n', None), (4, 1, 'bar\n', None)] - >>> gt_context_2.do_grep(StringIO(small_gap)) + >>> gt_context_2.do_grep(BytesIO(small_gap)) [(0, -1, 'bar\n', None), (1, -1, 'bar\n', None), (2, 0, 'foo\n', [(0, 3)]), (3, 1, 'bar\n', None), (4, 0, 'foo\n', [(0, 3)]), (5, 1, 'bar\n', None), (6, 1, 'bar\n', None)] - >>> gt_context_2.do_grep(StringIO(no_eol)) + >>> gt_context_2.do_grep(BytesIO(no_eol)) [(0, 0, 'foo', [(0, 3)])] - >>> gt_context_2.do_grep(StringIO(middle_of_line)) + >>> gt_context_2.do_grep(BytesIO(middle_of_line)) [(0, -1, 'bar\n', None), (1, -1, 'bar\n', None), (2, 0, 'barfoobar\n', [(3, 6)]), (3, 1, 'bar\n', None), (4, 1, 'bar\n', None)] 1 line of before-context, no lines after. >>> gt_before_context_1 = grin.GrepText(re.compile('foo'), options=grin.Options(before_context=1, after_context=0)) - >>> gt_before_context_1.do_grep(StringIO(all_foo)) + >>> gt_before_context_1.do_grep(BytesIO(all_foo)) [(0, 0, 'foo\n', [(0, 3)]), (1, 0, 'foo\n', [(0, 3)]), (2, 0, 'foo\n', [(0, 3)]), (3, 0, 'foo\n', [(0, 3)]), (4, 0, 'foo\n', [(0, 3)])] - >>> gt_before_context_1.do_grep(StringIO(first_foo)) + >>> gt_before_context_1.do_grep(BytesIO(first_foo)) [(0, 0, 'foo\n', [(0, 3)])] - >>> gt_before_context_1.do_grep(StringIO(last_foo)) + >>> gt_before_context_1.do_grep(BytesIO(last_foo)) [(3, -1, 'bar\n', None), (4, 0, 'foo\n', [(0, 3)])] - >>> gt_before_context_1.do_grep(StringIO(second_foo)) + >>> gt_before_context_1.do_grep(BytesIO(second_foo)) [(0, -1, 'bar\n', None), (1, 0, 'foo\n', [(0, 3)])] - >>> gt_before_context_1.do_grep(StringIO(second_last_foo)) + >>> gt_before_context_1.do_grep(BytesIO(second_last_foo)) [(2, -1, 'bar\n', None), (3, 0, 'foo\n', [(0, 3)])] - >>> gt_before_context_1.do_grep(StringIO(middle_foo)) + >>> gt_before_context_1.do_grep(BytesIO(middle_foo)) [(1, -1, 'bar\n', None), (2, 0, 'foo\n', [(0, 3)])] - >>> gt_before_context_1.do_grep(StringIO(small_gap)) + >>> gt_before_context_1.do_grep(BytesIO(small_gap)) [(1, -1, 'bar\n', None), (2, 0, 'foo\n', [(0, 3)]), (3, -1, 'bar\n', None), (4, 0, 'foo\n', [(0, 3)])] - >>> gt_before_context_1.do_grep(StringIO(no_eol)) + >>> gt_before_context_1.do_grep(BytesIO(no_eol)) [(0, 0, 'foo', [(0, 3)])] - >>> gt_before_context_1.do_grep(StringIO(middle_of_line)) + >>> gt_before_context_1.do_grep(BytesIO(middle_of_line)) [(1, -1, 'bar\n', None), (2, 0, 'barfoobar\n', [(3, 6)])] 1 line of after-context, no lines before. >>> gt_after_context_1 = grin.GrepText(re.compile('foo'), options=grin.Options(before_context=0, after_context=1)) - >>> gt_after_context_1.do_grep(StringIO(all_foo)) + >>> gt_after_context_1.do_grep(BytesIO(all_foo)) [(0, 0, 'foo\n', [(0, 3)]), (1, 0, 'foo\n', [(0, 3)]), (2, 0, 'foo\n', [(0, 3)]), (3, 0, 'foo\n', [(0, 3)]), (4, 0, 'foo\n', [(0, 3)])] - >>> gt_after_context_1.do_grep(StringIO(first_foo)) + >>> gt_after_context_1.do_grep(BytesIO(first_foo)) [(0, 0, 'foo\n', [(0, 3)]), (1, 1, 'bar\n', None)] - >>> gt_after_context_1.do_grep(StringIO(last_foo)) + >>> gt_after_context_1.do_grep(BytesIO(last_foo)) [(4, 0, 'foo\n', [(0, 3)])] - >>> gt_after_context_1.do_grep(StringIO(second_foo)) + >>> gt_after_context_1.do_grep(BytesIO(second_foo)) [(1, 0, 'foo\n', [(0, 3)]), (2, 1, 'bar\n', None)] - >>> gt_after_context_1.do_grep(StringIO(second_last_foo)) + >>> gt_after_context_1.do_grep(BytesIO(second_last_foo)) [(3, 0, 'foo\n', [(0, 3)]), (4, 1, 'bar\n', None)] - >>> gt_after_context_1.do_grep(StringIO(middle_foo)) + >>> gt_after_context_1.do_grep(BytesIO(middle_foo)) [(2, 0, 'foo\n', [(0, 3)]), (3, 1, 'bar\n', None)] - >>> gt_after_context_1.do_grep(StringIO(small_gap)) + >>> gt_after_context_1.do_grep(BytesIO(small_gap)) [(2, 0, 'foo\n', [(0, 3)]), (3, 1, 'bar\n', None), (4, 0, 'foo\n', [(0, 3)]), (5, 1, 'bar\n', None)] - >>> gt_after_context_1.do_grep(StringIO(no_eol)) + >>> gt_after_context_1.do_grep(BytesIO(no_eol)) [(0, 0, 'foo', [(0, 3)])] - >>> gt_after_context_1.do_grep(StringIO(middle_of_line)) + >>> gt_after_context_1.do_grep(BytesIO(middle_of_line)) [(2, 0, 'barfoobar\n', [(3, 6)]), (3, 1, 'bar\n', None)] ''' From 147b09ae7d161896e196d16dd860d84386871083 Mon Sep 17 00:00:00 2001 From: Matthew Brett Date: Tue, 6 Sep 2016 16:09:49 -0700 Subject: [PATCH 03/15] BF: always open files in binary mode Python 3 port assumes that input is always binary, and converts to latin1 before running regex. --- grin.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/grin.py b/grin.py index aa33bb2..17c8c1e 100755 --- a/grin.py +++ b/grin.py @@ -468,7 +468,7 @@ def report(self, context_lines, filename=None): color_substring = colorize(old_substring, **style) line = line[:start] + color_substring + line[end:] total_offset += len(color_substring) - len(old_substring) - + ns = dict( lineno = i+1, sep = {PRE: '-', POST: '+', MATCH: ':'}[kind], @@ -506,8 +506,8 @@ def grep_a_file(self, filename, opener=open): f = sys.stdin filename = '' else: - # 'r' does the right thing for both open ('rt') and gzip.open ('rb') - f = opener(filename, 'r') + # Always open in binary mode + f = opener(filename, 'rb') try: unique_context = self.do_grep(f) finally: From 3f0d16e7a6dc6fbf44e3042b506bfc1d309d55e3 Mon Sep 17 00:00:00 2001 From: Matthew Brett Date: Sat, 6 Apr 2019 15:44:20 +0100 Subject: [PATCH 04/15] Extend use of to_str to fix some crashes --- grin.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/grin.py b/grin.py index 17c8c1e..9781432 100755 --- a/grin.py +++ b/grin.py @@ -236,12 +236,12 @@ def read_block_with_context(self, prev, fp, fp_size): """ if fp_size is None: target_io_size = READ_BLOCKSIZE - block_main = fp.read(target_io_size) + block_main = to_str(fp.read(target_io_size)) is_last_block = len(block_main) < target_io_size else: remaining = max(fp_size - fp.tell(), 0) target_io_size = min(READ_BLOCKSIZE, remaining) - block_main = fp.read(target_io_size) + block_main = to_str(fp.read(target_io_size)) is_last_block = target_io_size == remaining if prev is None: @@ -250,7 +250,7 @@ def read_block_with_context(self, prev, fp, fp_size): # can avoid the overhead of locating lines of 'before' and # 'after' context. result = DataBlock( - data = to_str(block_main), + data = block_main, start = 0, end = len(block_main), before_count = 0, @@ -280,16 +280,17 @@ def read_block_with_context(self, prev, fp, fp_size): before_lines = prev.data[before_start:prev.end] # Using readline() to force this block out to a newline boundary... curr_block = (prev.data[prev.end:] + block_main + - ('' if is_last_block else fp.readline())) + ('' if is_last_block else to_str(fp.readline()))) # Read in some lines of 'after' context. if is_last_block: after_lines = '' else: - after_lines_list = [fp.readline() for i in range(self.options.after_context)] + after_lines_list = [to_str(fp.readline()) + for i in range(self.options.after_context)] after_lines = ''.join(after_lines_list) result = DataBlock( - data = to_str(before_lines + curr_block + after_lines), + data = before_lines + curr_block + after_lines, start = len(before_lines), end = len(before_lines) + len(curr_block), before_count = before_count, From 9f75595b6ef32c0545154bb11ef81141ca9a506e Mon Sep 17 00:00:00 2001 From: Lukasz Konopski Date: Mon, 15 Apr 2019 13:24:30 +0200 Subject: [PATCH 05/15] skip node_modules get rid of js garbage in results --- grin.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/grin.py b/grin.py index edf0784..d7f4676 100755 --- a/grin.py +++ b/grin.py @@ -816,7 +816,7 @@ def get_grin_arg_parser(parser=None): default=True, action='store_true', help="do skip .hidden directories [default]") parser.add_argument('-d', '--skip-dirs', - default='CVS,RCS,.svn,.hg,.bzr,build,dist,target', + default='CVS,RCS,.svn,.hg,.bzr,build,dist,target,node_modules', help="comma-separated list of directory names to skip [default=%(default)r]") parser.add_argument('-D', '--no-skip-dirs', dest='skip_dirs', action='store_const', const='', @@ -874,7 +874,7 @@ def get_grind_arg_parser(parser=None): default=True, action='store_true', help="do skip .hidden directories") parser.add_argument('-d', '--skip-dirs', - default='CVS,RCS,.svn,.hg,.bzr,build,dist,target', + default='CVS,RCS,.svn,.hg,.bzr,build,dist,target,node_modules', help="comma-separated list of directory names to skip [default=%(default)r]") parser.add_argument('-D', '--no-skip-dirs', dest='skip_dirs', action='store_const', const='', From d0457df0bd6e126b38ee626b06e8c1a7705637d8 Mon Sep 17 00:00:00 2001 From: Matthew Brett Date: Wed, 2 Dec 2020 17:54:46 +0000 Subject: [PATCH 06/15] Update for 1.3 release. --- ANNOUNCE.txt | 16 ++++------------ LICENSE.txt | 2 +- grin.py | 2 +- setup.cfg | 2 +- setup.py | 5 +++-- 5 files changed, 10 insertions(+), 17 deletions(-) diff --git a/ANNOUNCE.txt b/ANNOUNCE.txt index 3e74df2..e067eeb 100644 --- a/ANNOUNCE.txt +++ b/ANNOUNCE.txt @@ -2,18 +2,10 @@ grin is a grep-like tool for recursively searching through text files, primarily source code. Download: http://pypi.python.org/pypi/grin - SVN: https://svn.enthought.com/svn/sandbox/grin/trunk + Git: https://github.com/matthew-brett/grin License: BSD -grin 1.2.1 is a bug-fix release. +grin 1.30.0 is a feature release - * Windows defaults to not coloring the output. (Paul Pelzl) - report. - - * Fix the reading of gzip files. (Brandon Craig Rhodes) - - * Quit gracefully when piping to a program that exits prematurely. - (Brandon Craig Rhodes) - - * Sort the basenames of files during traversal in order to maintain - a repeatable ordering. (Brandon Craig Rhodes) + * Python 3 compatibility. + * Bugfix: Handle empty lists of skip-dirs and skip-exts. diff --git a/LICENSE.txt b/LICENSE.txt index 71a2e63..488f117 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -1,7 +1,7 @@ This software is OSI Certified Open Source Software. OSI Certified is a certification mark of the Open Source Initiative. -Copyright (c) 2007, Enthought, Inc. +Copyright (c) 2007-2020, Enthought, Inc. All rights reserved. Redistribution and use in source and binary forms, with or without diff --git a/grin.py b/grin.py index 9781432..598f2d2 100755 --- a/grin.py +++ b/grin.py @@ -24,7 +24,7 @@ ints2bytes = lambda ints : ''.join(map(chr, ints)) #### Constants #### -__version__ = '1.2.1' +__version__ = '1.3.0' # Maintain the numerical order of these constants. We use them for sorting. PRE = -1 diff --git a/setup.cfg b/setup.cfg index c089787..efebb72 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,5 @@ [egg_info] -tag_build = .dev +tag_build = [aliases] release = egg_info -RDb '' diff --git a/setup.py b/setup.py index 3d1f1eb..c340a2c 100644 --- a/setup.py +++ b/setup.py @@ -12,12 +12,12 @@ setup( name='grin', - version='1.2.1', + version='1.3.0', author='Robert Kern', author_email='robert.kern@enthought.com', description="A grep program configured the way I like it.", license="BSD", - url='https://github.com/rkern/grin', + url='https://github.com/matthew-brett/grin', classifiers=[ "License :: OSI Approved :: BSD License", "Development Status :: 5 - Production/Stable", @@ -25,6 +25,7 @@ "Intended Audience :: Developers", "Operating System :: OS Independent", "Programming Language :: Python", + "Programming Language :: Python :: 3", "Topic :: Utilities", ], py_modules=["grin"], From dd80f79ce7324b22a71ecb6e38bac665363498e5 Mon Sep 17 00:00:00 2001 From: Matthew Brett Date: Thu, 3 Dec 2020 00:43:03 +0000 Subject: [PATCH 07/15] Updated URLs to this fork. --- README.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.rst b/README.rst index 170fa70..ddca300 100644 --- a/README.rst +++ b/README.rst @@ -84,11 +84,11 @@ Running the unittests requires the nose_ framework:: The development sources are hosted on Github: - https://github.com/rkern/grin + https://github.com/matthew-brett/grin There is one little tweak to the installation that you may want to consider. By default, setuptools installs scripts indirectly; the scripts installed to -$prefix/bin or Python2x\Scripts use setuptools' pkg_resources module to load +$prefix/bin or Python\Scripts use setuptools' pkg_resources module to load the exact version of grin egg that installed the script, then runs the script's main() function. This is not usually a bad feature, but it can add substantial startup overhead for a small command-line utility like grin. If you want the @@ -241,4 +241,4 @@ Bugs and Such Please make a new issue at the Github issue tracker. - https://github.com/rkern/grin + https://github.com/matthew-brett/grin From b9c04e62a79345ab5901824caafaea7600c2d5aa Mon Sep 17 00:00:00 2001 From: Matthew Brett Date: Thu, 3 Dec 2020 00:46:39 +0000 Subject: [PATCH 08/15] Me maintainer --- setup.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/setup.py b/setup.py index c340a2c..8d4e7da 100644 --- a/setup.py +++ b/setup.py @@ -15,6 +15,8 @@ version='1.3.0', author='Robert Kern', author_email='robert.kern@enthought.com', + maintainer='Matthew Brett', + maintainer_email='matthew.brett@gmail.com', description="A grep program configured the way I like it.", license="BSD", url='https://github.com/matthew-brett/grin', From 77cf0cd6ac98eb3c97e6d16d1ea7eb5c619b20d9 Mon Sep 17 00:00:00 2001 From: Matthew Brett Date: Thu, 3 Dec 2020 10:28:17 +0000 Subject: [PATCH 09/15] Fix .travis.yml versions --- .travis.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index a2d9eb2..8df0d0f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,11 +1,11 @@ language: python python: - - 2.6 - 2.7 - - 3.3 - - 3.4 - 3.5 + - 3.6 + - 3.7 + - 3.8 install: - pip install -e . From 21af8addb1d141f61f10cc8bf400b85c8de97590 Mon Sep 17 00:00:00 2001 From: Matthew Brett Date: Mon, 7 Dec 2020 15:11:49 +0000 Subject: [PATCH 10/15] RELEASING file. --- RELEASING.txt | 49 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 RELEASING.txt diff --git a/RELEASING.txt b/RELEASING.txt new file mode 100644 index 0000000..bc24814 --- /dev/null +++ b/RELEASING.txt @@ -0,0 +1,49 @@ +# Release process + +Check log since last release, build `ChangeLog`. + +``` +git clean -fxd +``` + +## Do pre-release + +Change `tag_build` to `rc` in `setup.cfg`. + +Check travis-ci. + +``` +git clean -fxd +python setup.py sdist +twine upload sdist/grin*.tar.gz +``` + +As for feedback. + +Change `tag_build` to empty in `setup.cfg`. + +Check travis-ci. + +``` +git clean -fxd +python setup.py sdist +twine upload sdist/grin*.tar.gz +``` + +``` +git tag -S +git push --tags +``` + +## Prepare for next release + +Change `__version__` in `grin.py`. + +Change `__version__` in `setup.py`. + +Change `tag_build` to `.dev` in `setup.cfg`. + +``` +git commit +git push +``` From d1862661d9a4c2d232fbea57d7bd8a6845a46f38 Mon Sep 17 00:00:00 2001 From: Matthew Brett Date: Mon, 7 Dec 2020 15:15:30 +0000 Subject: [PATCH 11/15] Pre-release --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index efebb72..6048ffb 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,5 @@ [egg_info] -tag_build = +tag_build = .rc [aliases] release = egg_info -RDb '' From 729e2ce3124ff2be632de3e58e8e796055dde886 Mon Sep 17 00:00:00 2001 From: Matthew Brett Date: Mon, 7 Dec 2020 15:22:33 +0000 Subject: [PATCH 12/15] Ready for 1.3.0 release --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 6048ffb..e6efd10 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,5 @@ [egg_info] -tag_build = .rc +tag_build = [aliases] release = egg_info -RDb '' From e9fd1f456535a298211730839010f35f09c33759 Mon Sep 17 00:00:00 2001 From: Matthew Brett Date: Mon, 7 Dec 2020 15:25:12 +0000 Subject: [PATCH 13/15] Fix git tag instructions --- RELEASING.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RELEASING.txt b/RELEASING.txt index bc24814..90b7cb0 100644 --- a/RELEASING.txt +++ b/RELEASING.txt @@ -31,7 +31,7 @@ twine upload sdist/grin*.tar.gz ``` ``` -git tag -S +git tag -s git push --tags ``` From 857e418ab98f4269b6082cbbaf80e96e5924af92 Mon Sep 17 00:00:00 2001 From: Matthew Brett Date: Mon, 7 Dec 2020 15:25:25 +0000 Subject: [PATCH 14/15] Complete post-release procedure. --- grin.py | 2 +- setup.cfg | 2 +- setup.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/grin.py b/grin.py index 598f2d2..5222eb3 100755 --- a/grin.py +++ b/grin.py @@ -24,7 +24,7 @@ ints2bytes = lambda ints : ''.join(map(chr, ints)) #### Constants #### -__version__ = '1.3.0' +__version__ = '1.3.1' # Maintain the numerical order of these constants. We use them for sorting. PRE = -1 diff --git a/setup.cfg b/setup.cfg index e6efd10..c089787 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,5 @@ [egg_info] -tag_build = +tag_build = .dev [aliases] release = egg_info -RDb '' diff --git a/setup.py b/setup.py index 8d4e7da..7fef44b 100644 --- a/setup.py +++ b/setup.py @@ -12,7 +12,7 @@ setup( name='grin', - version='1.3.0', + version='1.3.1', author='Robert Kern', author_email='robert.kern@enthought.com', maintainer='Matthew Brett', From fa2c4d2c49a788bdc489514a0e0bcadf219d8fa9 Mon Sep 17 00:00:00 2001 From: Matthew Brett Date: Tue, 8 Dec 2020 10:05:11 +0000 Subject: [PATCH 15/15] Add MANIFEST.in to add license file. --- MANIFEST.in | 1 + 1 file changed, 1 insertion(+) create mode 100644 MANIFEST.in diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..42eb410 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1 @@ +include LICENSE.txt