⚡️ Speed up method FixedWidthReader.get_rows by 15%

[codeflash-ai]

📄 15% (0.15x) speedup for FixedWidthReader.get_rows in pandas/io/parsers/python_parser.py

⏱️ Runtime : 1.18 milliseconds → 1.03 milliseconds (best of 236 runs)

📝 Explanation and details

The optimized version achieves a 15% speedup through several micro-optimizations in the get_rows method's main loop:

Key Optimizations:

1. Local method caching: Stores buffer_rows.append and detect_rows.append as local variables (append_buffer, append_detect). This avoids repeated attribute lookups in the loop, reducing overhead from ~279ns to ~205ns per append operation.

2. Streamlined skiprows handling: Replaces the conditional if skiprows is None: skiprows = set() with a single assignment skipset = skiprows if skiprows is not None else set(). This eliminates the branch and potential set creation when skiprows is already provided.

3. Optimized loop structure: Introduces a count variable to track valid (non-skipped) rows, allowing the break condition to check count >= infer_nrows instead of calling len(detect_rows) repeatedly. This removes the overhead of list length calculation (~304ns per check).

Performance Impact:

• Small files (< 20 rows): Shows 7-11% slowdown due to additional variable initialization overhead
• Large files (1000+ rows): Shows 12-43% speedup, with the best gains when skiprows filters out many early rows
• The optimization particularly benefits scenarios with substantial skiprows usage, where the count-based early exit prevents unnecessary iterations

Why it works:
In Python, attribute lookups (obj.method) and function calls (len()) have measurable overhead in tight loops. By caching method references locally and using direct counter comparisons instead of list length checks, the optimization reduces per-iteration overhead from ~1330ns to ~1180ns total in the main loop body, compounding significantly over large datasets.

✅ Correctness verification report:

Test	Status
⚙️ Existing Unit Tests	🔘 None Found
🌀 Generated Regression Tests	✅ 102 Passed
⏪ Replay Tests	🔘 None Found
🔎 Concolic Coverage Tests	🔘 None Found
📊 Tests Coverage	100.0%

🌀 Generated Regression Tests and Runtime

from __future__ import annotations

import re
from collections import abc
from collections.abc import Iterator
from io import StringIO
from typing import IO, Literal

import numpy as np
# imports
import pytest
from pandas.io.parsers.python_parser import FixedWidthReader

# unit tests

# ---------------- BASIC TEST CASES ----------------

def test_get_rows_basic_no_skip():
    # Basic: 3 lines, no skip, infer_nrows = 2
    data = "abcde\nfghij\nklmno\n"
    f = StringIO(data)
    reader = FixedWidthReader(f, [(0,2), (2,5)], delimiter=None, comment=None)
    codeflash_output = reader.get_rows(2); rows = codeflash_output # 2.61μs -> 2.92μs (10.4% slower)

def test_get_rows_basic_skip_one():
    # Basic: 3 lines, skip line 1, infer_nrows = 2
    data = "abcde\nfghij\nklmno\n"
    f = StringIO(data)
    reader = FixedWidthReader(f, [(0,2), (2,5)], delimiter=None, comment=None)
    codeflash_output = reader.get_rows(2, skiprows={1}); rows = codeflash_output # 2.66μs -> 2.88μs (7.64% slower)

def test_get_rows_basic_skip_multiple():
    # Basic: 5 lines, skip 1 and 3, infer_nrows = 2
    data = "a\nb\nc\nd\ne\n"
    f = StringIO(data)
    reader = FixedWidthReader(f, [(0,1)], delimiter=None, comment=None)
    codeflash_output = reader.get_rows(2, skiprows={1,3}); rows = codeflash_output # 2.46μs -> 2.66μs (7.41% slower)

def test_get_rows_basic_skip_none_equivalent():
    # skiprows=None and skiprows=set() should be equivalent
    data = "x\ny\nz\n"
    f1 = StringIO(data)
    f2 = StringIO(data)
    reader1 = FixedWidthReader(f1, [(0,1)], delimiter=None, comment=None)
    reader2 = FixedWidthReader(f2, [(0,1)], delimiter=None, comment=None)
    codeflash_output = reader1.get_rows(2); rows1 = codeflash_output # 2.22μs -> 2.48μs (10.7% slower)
    codeflash_output = reader2.get_rows(2, skiprows=set()); rows2 = codeflash_output # 1.21μs -> 1.22μs (0.980% slower)

def test_get_rows_basic_exact_rows():
    # infer_nrows matches number of data rows
    data = "row1\nrow2\nrow3\n"
    f = StringIO(data)
    reader = FixedWidthReader(f, [(0,2)], delimiter=None, comment=None)
    codeflash_output = reader.get_rows(3); rows = codeflash_output # 2.42μs -> 2.65μs (8.54% slower)

# ---------------- EDGE TEST CASES ----------------

def test_get_rows_empty_file():
    # Edge: Empty file
    data = ""
    f = StringIO(data)
    reader = FixedWidthReader(f, [(0,2)], delimiter=None, comment=None)
    codeflash_output = reader.get_rows(2); rows = codeflash_output # 1.61μs -> 1.74μs (7.20% slower)

def test_get_rows_skip_all():
    # Edge: skiprows skips all lines
    data = "a\nb\nc\n"
    f = StringIO(data)
    reader = FixedWidthReader(f, [(0,1)], delimiter=None, comment=None)
    codeflash_output = reader.get_rows(2, skiprows={0,1,2}); rows = codeflash_output # 2.47μs -> 2.45μs (0.815% faster)

def test_get_rows_infer_nrows_zero():
    # Edge: infer_nrows=0 should return []
    data = "abc\n"
    f = StringIO(data)
    reader = FixedWidthReader(f, [(0,1)], delimiter=None, comment=None)
    codeflash_output = reader.get_rows(0); rows = codeflash_output # 2.00μs -> 2.21μs (9.58% slower)

def test_get_rows_skiprows_beyond_length():
    # Edge: skiprows includes indices beyond file length
    data = "a\nb\n"
    f = StringIO(data)
    reader = FixedWidthReader(f, [(0,1)], delimiter=None, comment=None)
    codeflash_output = reader.get_rows(2, skiprows={10,11,12}); rows = codeflash_output # 2.29μs -> 2.56μs (10.7% slower)

def test_get_rows_infer_nrows_gt_lines():
    # Edge: infer_nrows > number of available, should return all
    data = "a\nb\n"
    f = StringIO(data)
    reader = FixedWidthReader(f, [(0,1)], delimiter=None, comment=None)
    codeflash_output = reader.get_rows(5); rows = codeflash_output # 2.27μs -> 2.43μs (6.54% slower)

def test_get_rows_skiprows_and_short_file():
    # Edge: skiprows leaves fewer than infer_nrows
    data = "x\ny\nz\n"
    f = StringIO(data)
    reader = FixedWidthReader(f, [(0,1)], delimiter=None, comment=None)
    codeflash_output = reader.get_rows(2, skiprows={0,1}); rows = codeflash_output # 2.44μs -> 2.49μs (1.93% slower)

def test_get_rows_skiprows_negative_index():
    # Edge: skiprows contains negative index (should not skip any lines)
    data = "a\nb\n"
    f = StringIO(data)
    reader = FixedWidthReader(f, [(0,1)], delimiter=None, comment=None)
    codeflash_output = reader.get_rows(2, skiprows={-1}); rows = codeflash_output # 2.19μs -> 2.35μs (7.18% slower)

def test_get_rows_handles_blank_lines():
    # Edge: file contains blank lines
    data = "a\n\nb\n\nc\n"
    f = StringIO(data)
    reader = FixedWidthReader(f, [(0,1)], delimiter=None, comment=None)
    codeflash_output = reader.get_rows(4); rows = codeflash_output # 2.59μs -> 2.79μs (7.03% slower)

def test_get_rows_handles_carriage_return():
    # Edge: file uses \r\n line endings
    data = "a\r\nb\r\nc\r\n"
    f = StringIO(data)
    reader = FixedWidthReader(f, [(0,1)], delimiter=None, comment=None)
    codeflash_output = reader.get_rows(2); rows = codeflash_output # 2.28μs -> 2.57μs (11.2% slower)

def test_get_rows_handles_trailing_newlines():
    # Edge: file ends with multiple newlines
    data = "x\ny\nz\n\n\n"
    f = StringIO(data)
    reader = FixedWidthReader(f, [(0,1)], delimiter=None, comment=None)
    codeflash_output = reader.get_rows(5); rows = codeflash_output # 2.72μs -> 2.96μs (8.36% slower)

# ---------------- LARGE SCALE TEST CASES ----------------

def test_get_rows_large_file_no_skip():
    # Large: 1000 lines, no skip, infer_nrows=1000
    data = "".join(f"{i:04d}\n" for i in range(1000))
    f = StringIO(data)
    reader = FixedWidthReader(f, [(0,4)], delimiter=None, comment=None)
    codeflash_output = reader.get_rows(1000); rows = codeflash_output # 100μs -> 89.2μs (13.1% faster)

def test_get_rows_large_file_with_skips():
    # Large: 1000 lines, skip every 10th, infer_nrows=900
    data = "".join(f"{i:04d}\n" for i in range(1000))
    skiprows = set(range(0, 1000, 10))
    f = StringIO(data)
    reader = FixedWidthReader(f, [(0,4)], delimiter=None, comment=None)
    codeflash_output = reader.get_rows(900, skiprows=skiprows); rows = codeflash_output # 103μs -> 89.3μs (16.3% faster)
    # Should skip 100 lines, return next 900
    expected = [f"{i:04d}\n" for i in range(1000) if i not in skiprows][:900]

def test_get_rows_large_file_infer_nrows_small():
    # Large: 1000 lines, infer_nrows=10
    data = "".join(f"{i:04d}\n" for i in range(1000))
    f = StringIO(data)
    reader = FixedWidthReader(f, [(0,4)], delimiter=None, comment=None)
    codeflash_output = reader.get_rows(10); rows = codeflash_output # 3.37μs -> 3.57μs (5.69% slower)

def test_get_rows_large_file_skip_last():
    # Large: 1000 lines, skip last 10, infer_nrows=990
    data = "".join(f"{i:04d}\n" for i in range(1000))
    skiprows = set(range(990, 1000))
    f = StringIO(data)
    reader = FixedWidthReader(f, [(0,4)], delimiter=None, comment=None)
    codeflash_output = reader.get_rows(990, skiprows=skiprows); rows = codeflash_output # 102μs -> 90.7μs (13.5% faster)
    expected = [f"{i:04d}\n" for i in range(990)]

def test_get_rows_large_file_skip_first():
    # Large: 1000 lines, skip first 500, infer_nrows=500
    data = "".join(f"{i:04d}\n" for i in range(1000))
    skiprows = set(range(500))
    f = StringIO(data)
    reader = FixedWidthReader(f, [(0,4)], delimiter=None, comment=None)
    codeflash_output = reader.get_rows(500, skiprows=skiprows); rows = codeflash_output # 92.7μs -> 73.1μs (26.8% faster)
    expected = [f"{i:04d}\n" for i in range(500, 1000)]
# codeflash_output is used to check that the output of the original code is the same as that of the optimized code.
#------------------------------------------------
from __future__ import annotations

import re
from collections import abc
from collections.abc import Iterator
from io import StringIO
from typing import IO, Literal

import numpy as np
# imports
import pytest
from pandas.io.parsers.python_parser import FixedWidthReader

# unit tests

# ------------------------
# BASIC TEST CASES
# ------------------------

def test_basic_no_skiprows_reads_n_lines():
    # Test that get_rows returns the correct number of lines, no skiprows
    data = "abcde\nfghij\nklmno\npqrst\n"
    f = StringIO(data)
    rdr = FixedWidthReader(f, [(0,2),(2,5)], delimiter=None, comment=None)
    codeflash_output = rdr.get_rows(2); rows = codeflash_output # 2.40μs -> 2.68μs (10.4% slower)
    # Should not read more than infer_nrows lines

def test_basic_skiprows_skips_correct_lines():
    # Test that get_rows skips the correct lines
    data = "row0\nrow1\nrow2\nrow3\n"
    f = StringIO(data)
    rdr = FixedWidthReader(f, [(0,2)], delimiter=None, comment=None)
    codeflash_output = rdr.get_rows(2, skiprows={1}); rows = codeflash_output # 2.54μs -> 2.68μs (5.22% slower)

def test_basic_skiprows_multiple():
    # Test skipping multiple lines
    data = "a\nb\nc\nd\ne\n"
    f = StringIO(data)
    rdr = FixedWidthReader(f, [(0,1)], delimiter=None, comment=None)
    codeflash_output = rdr.get_rows(2, skiprows={0,2,4}); rows = codeflash_output # 2.60μs -> 2.59μs (0.463% faster)

def test_basic_skiprows_empty_set():
    # skiprows as empty set should behave as no skiprows
    data = "a\nb\nc\n"
    f = StringIO(data)
    rdr = FixedWidthReader(f, [(0,1)], delimiter=None, comment=None)
    codeflash_output = rdr.get_rows(2); rows1 = codeflash_output # 2.18μs -> 2.47μs (11.6% slower)
    f.seek(0)
    codeflash_output = rdr.get_rows(2, skiprows=set()); rows2 = codeflash_output # 1.25μs -> 1.44μs (13.2% slower)

def test_basic_skiprows_none_equivalent_to_empty():
    # skiprows=None should be equivalent to skiprows=set()
    data = "a\nb\nc\n"
    f = StringIO(data)
    rdr = FixedWidthReader(f, [(0,1)], delimiter=None, comment=None)
    codeflash_output = rdr.get_rows(2); rows1 = codeflash_output # 2.12μs -> 2.27μs (6.49% slower)
    f.seek(0)
    codeflash_output = rdr.get_rows(2, skiprows=None); rows2 = codeflash_output # 1.35μs -> 1.53μs (11.9% slower)

def test_basic_reads_less_than_n_if_file_short():
    # Should return all lines if file has fewer than infer_nrows lines
    data = "x\ny\n"
    f = StringIO(data)
    rdr = FixedWidthReader(f, [(0,1)], delimiter=None, comment=None)
    codeflash_output = rdr.get_rows(10); rows = codeflash_output # 2.10μs -> 2.29μs (8.38% slower)

def test_basic_zero_infer_nrows_returns_empty():
    # infer_nrows=0 should return empty list
    data = "a\nb\nc\n"
    f = StringIO(data)
    rdr = FixedWidthReader(f, [(0,1)], delimiter=None, comment=None)
    codeflash_output = rdr.get_rows(0); rows = codeflash_output # 1.87μs -> 2.05μs (8.88% slower)

# ------------------------
# EDGE TEST CASES
# ------------------------

def test_edge_empty_file_returns_empty():
    # Empty file should return empty list
    data = ""
    f = StringIO(data)
    rdr = FixedWidthReader(f, [(0,1)], delimiter=None, comment=None)
    codeflash_output = rdr.get_rows(5); rows = codeflash_output # 1.55μs -> 1.72μs (9.86% slower)

def test_edge_skiprows_all_lines():
    # skiprows skips all lines: should return empty list
    data = "1\n2\n3\n"
    f = StringIO(data)
    rdr = FixedWidthReader(f, [(0,1)], delimiter=None, comment=None)
    codeflash_output = rdr.get_rows(2, skiprows={0,1,2}); rows = codeflash_output # 2.51μs -> 2.31μs (8.74% faster)

def test_edge_skiprows_some_out_of_bounds():
    # skiprows contains indices that are out of bounds: should ignore them
    data = "a\nb\n"
    f = StringIO(data)
    rdr = FixedWidthReader(f, [(0,1)], delimiter=None, comment=None)
    codeflash_output = rdr.get_rows(1, skiprows={0,5,10}); rows = codeflash_output # 2.21μs -> 2.29μs (3.44% slower)

def test_edge_skiprows_negative_index():
    # skiprows contains negative index: should never match any line
    data = "x\ny\n"
    f = StringIO(data)
    rdr = FixedWidthReader(f, [(0,1)], delimiter=None, comment=None)
    codeflash_output = rdr.get_rows(1, skiprows={-1}); rows = codeflash_output # 1.93μs -> 2.12μs (9.18% slower)

def test_edge_skiprows_beyond_file_length():
    # skiprows contains indices greater than file length
    data = "a\nb\n"
    f = StringIO(data)
    rdr = FixedWidthReader(f, [(0,1)], delimiter=None, comment=None)
    codeflash_output = rdr.get_rows(2, skiprows={100,101}); rows = codeflash_output # 2.20μs -> 2.38μs (7.81% slower)

def test_edge_infer_nrows_larger_than_file():
    # infer_nrows is much larger than file length
    data = "a\nb\n"
    f = StringIO(data)
    rdr = FixedWidthReader(f, [(0,1)], delimiter=None, comment=None)
    codeflash_output = rdr.get_rows(100); rows = codeflash_output # 2.23μs -> 2.55μs (12.7% slower)

def test_edge_file_with_blank_lines():
    # File with blank lines
    data = "a\n\nb\n\nc\n"
    f = StringIO(data)
    rdr = FixedWidthReader(f, [(0,1)], delimiter=None, comment=None)
    codeflash_output = rdr.get_rows(4); rows = codeflash_output # 2.58μs -> 2.83μs (8.74% slower)

def test_edge_file_with_only_blank_lines():
    # File with only blank lines
    data = "\n\n\n"
    f = StringIO(data)
    rdr = FixedWidthReader(f, [(0,1)], delimiter=None, comment=None)
    codeflash_output = rdr.get_rows(2); rows = codeflash_output # 2.19μs -> 2.40μs (8.87% slower)

def test_edge_file_with_long_lines():
    # File with very long lines
    data = "a"*1000 + "\n" + "b"*1000 + "\n"
    f = StringIO(data)
    rdr = FixedWidthReader(f, [(0,1000)], delimiter=None, comment=None)
    codeflash_output = rdr.get_rows(2); rows = codeflash_output # 3.70μs -> 3.90μs (5.11% slower)

def test_edge_file_with_no_newline_at_end():
    # File with no newline at end
    data = "a\nb\nc"
    f = StringIO(data)
    rdr = FixedWidthReader(f, [(0,1)], delimiter=None, comment=None)
    codeflash_output = rdr.get_rows(3); rows = codeflash_output # 2.45μs -> 2.74μs (10.5% slower)

def test_edge_skiprows_and_short_file():
    # skiprows skips some, but not all, and file is short
    data = "x\ny\nz\n"
    f = StringIO(data)
    rdr = FixedWidthReader(f, [(0,1)], delimiter=None, comment=None)
    codeflash_output = rdr.get_rows(2, skiprows={0,2}); rows = codeflash_output # 2.61μs -> 2.67μs (2.28% slower)

def test_edge_skiprows_and_zero_infer_nrows():
    # skiprows present but infer_nrows=0
    data = "a\nb\nc\n"
    f = StringIO(data)
    rdr = FixedWidthReader(f, [(0,1)], delimiter=None, comment=None)
    codeflash_output = rdr.get_rows(0, skiprows={1}); rows = codeflash_output # 2.06μs -> 2.25μs (8.48% slower)

def test_edge_skiprows_larger_than_infer_nrows():
    # skiprows skips more lines than infer_nrows, but file is long
    data = "".join(f"{i}\n" for i in range(10))
    f = StringIO(data)
    rdr = FixedWidthReader(f, [(0,1)], delimiter=None, comment=None)
    codeflash_output = rdr.get_rows(2, skiprows={0,1,2,3,4,5,6,7,8}); rows = codeflash_output # 3.18μs -> 3.01μs (5.55% faster)

# ------------------------
# LARGE SCALE TEST CASES
# ------------------------

def test_large_scale_reads_many_lines():
    # Test reading a large number of lines
    lines = [f"{i:04d}\n" for i in range(1000)]
    data = "".join(lines)
    f = StringIO(data)
    rdr = FixedWidthReader(f, [(0,4)], delimiter=None, comment=None)
    codeflash_output = rdr.get_rows(1000); rows = codeflash_output # 125μs -> 111μs (12.7% faster)

def test_large_scale_reads_subset_with_skiprows():
    # Test reading a subset with skiprows in a large file
    lines = [f"{i:04d}\n" for i in range(1000)]
    data = "".join(lines)
    f = StringIO(data)
    skip = set(range(0,1000,2))  # skip even lines
    rdr = FixedWidthReader(f, [(0,4)], delimiter=None, comment=None)
    codeflash_output = rdr.get_rows(500, skiprows=skip); rows = codeflash_output # 99.9μs -> 78.5μs (27.3% faster)
    expected = [f"{i:04d}\n" for i in range(1,1000,2)]

def test_large_scale_reads_until_infer_nrows():
    # Test that reading stops at infer_nrows, even if file is much larger
    lines = [f"line{i}\n" for i in range(1000)]
    data = "".join(lines)
    f = StringIO(data)
    rdr = FixedWidthReader(f, [(0,6)], delimiter=None, comment=None)
    codeflash_output = rdr.get_rows(100); rows = codeflash_output # 13.0μs -> 12.0μs (8.04% faster)

def test_large_scale_skiprows_last_lines():
    # Skip last 10 lines, should not affect reading first N lines
    lines = [f"{i:04d}\n" for i in range(1000)]
    data = "".join(lines)
    f = StringIO(data)
    skip = set(range(990,1000))
    rdr = FixedWidthReader(f, [(0,4)], delimiter=None, comment=None)
    codeflash_output = rdr.get_rows(990, skiprows=skip); rows = codeflash_output # 104μs -> 93.8μs (11.8% faster)

def test_large_scale_skiprows_first_lines():
    # Skip first 10 lines, should start reading from line 10
    lines = [f"{i:04d}\n" for i in range(1000)]
    data = "".join(lines)
    f = StringIO(data)
    skip = set(range(10))
    rdr = FixedWidthReader(f, [(0,4)], delimiter=None, comment=None)
    codeflash_output = rdr.get_rows(990, skiprows=skip); rows = codeflash_output # 104μs -> 92.3μs (12.7% faster)

def test_large_scale_skiprows_sparse():
    # Skip every 10th line, should read 900 lines out of 1000
    lines = [f"{i:04d}\n" for i in range(1000)]
    data = "".join(lines)
    f = StringIO(data)
    skip = set(range(0,1000,10))
    rdr = FixedWidthReader(f, [(0,4)], delimiter=None, comment=None)
    codeflash_output = rdr.get_rows(900, skiprows=skip); rows = codeflash_output # 104μs -> 91.0μs (14.7% faster)
    expected = [f"{i:04d}\n" for i in range(1000) if i%10!=0][:900]

def test_large_scale_skiprows_all_but_one():
    # Skip all but one line in a large file
    lines = [f"{i:04d}\n" for i in range(1000)]
    data = "".join(lines)
    f = StringIO(data)
    skip = set(range(999))
    rdr = FixedWidthReader(f, [(0,4)], delimiter=None, comment=None)
    codeflash_output = rdr.get_rows(1, skiprows=skip); rows = codeflash_output # 85.2μs -> 59.6μs (42.9% faster)

def test_large_scale_file_shorter_than_infer_nrows():
    # File shorter than infer_nrows, should return all lines
    lines = [f"{i:04d}\n" for i in range(500)]
    data = "".join(lines)
    f = StringIO(data)
    rdr = FixedWidthReader(f, [(0,4)], delimiter=None, comment=None)
    codeflash_output = rdr.get_rows(1000); rows = codeflash_output # 52.2μs -> 46.2μs (13.0% faster)

def test_large_scale_skiprows_empty_file():
    # Large skiprows set, but file is empty
    data = ""
    f = StringIO(data)
    skip = set(range(1000))
    rdr = FixedWidthReader(f, [(0,4)], delimiter=None, comment=None)
    codeflash_output = rdr.get_rows(1000, skiprows=skip); rows = codeflash_output # 1.79μs -> 1.96μs (8.58% slower)
# codeflash_output is used to check that the output of the original code is the same as that of the optimized code.

To edit these changes git checkout codeflash/optimize-FixedWidthReader.get_rows-mhog5z1l and push.