Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 28 additions & 0 deletions src/base/tests/test_util.py
Original file line number Diff line number Diff line change
Expand Up @@ -773,6 +773,34 @@ def _get_cr(self):
self.addCleanup(cr.close)
return cr

def test_explode_format_parallel_filter(self):
cr = self._get_cr()

cr.execute("SELECT MIN(id) FROM res_users")
min_id = cr.fetchone()[0]

q1 = "SELECT '{} {0} {x} {x:*^30d} {x.a.b} {x[0]}, {x[1]!s:*^30} {{x}}' FROM res_users"
q2 = "SELECT '{} {0} {x} {x:*^30d} {x.a.b} {x[0]}, {x[1]!s:*^30} {{x}}' FROM res_users WHERE {parallel_filter}"
expected_out = q1 + f" WHERE res_users.id BETWEEN {min_id} AND {min_id}"

out1 = util.explode_query_range(
cr,
q1,
table="res_users",
bucket_size=1,
format=False,
)[0]
self.assertEqual(out1, expected_out)

out2 = util.explode_query_range(
cr,
q2,
table="res_users",
bucket_size=1,
format=False,
)[0]
self.assertEqual(out2, expected_out)

def test_explode_mult_filters(self):
cr = self._get_cr()
queries = util.explode_query_range(
Expand Down
23 changes: 18 additions & 5 deletions src/util/pg.py
Original file line number Diff line number Diff line change
Expand Up @@ -260,7 +260,7 @@ def explode_query(cr, query, alias=None, num_buckets=8, prefix=None):
return [cr.mogrify(query, [num_buckets, index]).decode() for index in range(num_buckets)]


def explode_query_range(cr, query, table, alias=None, bucket_size=DEFAULT_BUCKET_SIZE, prefix=None):
def explode_query_range(cr, query, table, alias=None, bucket_size=DEFAULT_BUCKET_SIZE, prefix=None, format=True):
"""
Explode a query to multiple queries that can be executed in parallel.

Expand Down Expand Up @@ -335,16 +335,27 @@ def explode_query_range(cr, query, table, alias=None, bucket_size=DEFAULT_BUCKET
# Still, since the query may only be valid if there is no split, we force the usage of `prefix` in the query to
# validate its correctness and avoid scripts that pass the CI but fail in production.
parallel_filter = "{alias}.id IS NOT NULL".format(alias=alias)
return [query.format(parallel_filter=parallel_filter)]
return [
(
query.format(parallel_filter=parallel_filter)
if format
else query.replace("{parallel_filter}", parallel_filter)
)
]

parallel_filter = "{alias}.id BETWEEN %(lower-bound)s AND %(upper-bound)s".format(alias=alias)
query = query.replace("%", "%%").format(parallel_filter=parallel_filter)

query = query.replace("%", "%%")
query = (
query.format(parallel_filter=parallel_filter) if format else query.replace("{parallel_filter}", parallel_filter)
)

return [
cr.mogrify(query, {"lower-bound": ids[i], "upper-bound": ids[i + 1] - 1}).decode() for i in range(len(ids) - 1)
]


def explode_execute(cr, query, table, alias=None, bucket_size=DEFAULT_BUCKET_SIZE, logger=_logger):
def explode_execute(cr, query, table, alias=None, bucket_size=DEFAULT_BUCKET_SIZE, logger=_logger, format=True):
"""
Execute a query in parallel.

Expand Down Expand Up @@ -376,6 +387,8 @@ def explode_execute(cr, query, table, alias=None, bucket_size=DEFAULT_BUCKET_SIZ
:param int bucket_size: size of the buckets of ids to split the processing
:param logger: logger used to report the progress
:type logger: :class:`logging.Logger`
:param bool format: whether to use `.format` (instead of `.replace`) to replace the parallel filter,
setting it to `False` can prevent issues with hard-coded curly braces.
:return: the sum of `cr.rowcount` for each query run
:rtype: int

Expand All @@ -387,7 +400,7 @@ def explode_execute(cr, query, table, alias=None, bucket_size=DEFAULT_BUCKET_SIZ
"""
return parallel_execute(
cr,
explode_query_range(cr, query, table, alias=alias, bucket_size=bucket_size),
explode_query_range(cr, query, table, alias=alias, bucket_size=bucket_size, format=format),
logger=logger,
)

Expand Down