From 2c30de8a55aafd99c64d9f776d612cc220916078 Mon Sep 17 00:00:00 2001 From: Avi Avni Date: Mon, 6 Dec 2021 09:47:56 +0200 Subject: [PATCH 1/2] initial work on profiled execution plan --- redisgraph/execution_plan.py | 32 ++++++++++++++++++++----- redisgraph/graph.py | 15 ++++++++++++ tests/functional/test_all.py | 46 ++++++++++++++++++++++++++++++++++++ 3 files changed, 87 insertions(+), 6 deletions(-) diff --git a/redisgraph/execution_plan.py b/redisgraph/execution_plan.py index c13bbfb..bea2aec 100644 --- a/redisgraph/execution_plan.py +++ b/redisgraph/execution_plan.py @@ -1,18 +1,29 @@ +import re + + +class ProfileStats: + def __init__(self, records_produced, execution_time): + self.records_produced = records_produced + self.execution_time = execution_time + + class Operation: """ Operation, single operation within execution plan. """ - def __init__(self, name, args=None): + def __init__(self, name, args=None, profile_stats=None): """ Create a new operation. Args: name: string that represents the name of the operation args: operation arguments + profile_stats: profile statistics """ self.name = name self.args = args + self.profile_stats = profile_stats self.children = [] def append_child(self, child): @@ -32,7 +43,7 @@ def __eq__(self, o: object) -> bool: return (self.name == o.name and self.args == o.args) def __str__(self) -> str: - args_str = "" if self.args is None else f" | {self.args}" + args_str = "" if self.args is None else " | " + " | ".join(self.args) return f"{self.name}{args_str}" @@ -131,6 +142,17 @@ def _operation_tree(self): stack = [] current = None + def _create_operation(args): + profile_stats = None + name = args[0].strip() + args.pop(0) + if len(args) > 0 and "Records produced" in args[-1]: + records_produced = int(re.search("Records produced: (\\d+)", args[-1]).group(1)) + execution_time = float(re.search("Execution time: (\\d+.\\d+) ms", args[-1]).group(1)) + profile_stats = ProfileStats(records_produced, execution_time) + args.pop(-1) + return Operation(name, None if len(args) == 0 else [arg.strip() for arg in args], profile_stats) + # iterate plan operations while i < len(self.plan): current_op = self.plan[i] @@ -138,14 +160,12 @@ def _operation_tree(self): if op_level == level: # if the operation level equal to the current level # set the current operation and move next - args = current_op.split("|") - current = Operation(args[0].strip(), None if len(args) == 1 else args[1].strip()) + current = _create_operation(current_op.split("|")) i += 1 elif op_level == level + 1: # if the operation is child of the current operation # add it as child and set as current operation - args = current_op.split("|") - child = Operation(args[0].strip(), None if len(args) == 1 else args[1].strip()) + child = _create_operation(current_op.split("|")) current.append_child(child) stack.append(current) current = child diff --git a/redisgraph/graph.py b/redisgraph/graph.py index aecb97a..7dd832a 100644 --- a/redisgraph/graph.py +++ b/redisgraph/graph.py @@ -231,6 +231,21 @@ def execution_plan(self, query, params=None): plan = self.redis_con.execute_command("GRAPH.EXPLAIN", self.name, query) return ExecutionPlan(plan) + def profile(self, query, params=None): + """ + Get the profield execution plan for given query, + GRAPH.PROFILE returns an array of operations. + + Args: + query: the query that will be executed + params: query parameters + """ + if params is not None: + query = self._build_params_header(params) + query + + plan = self.redis_con.execute_command("GRAPH.PROFILE", self.name, query) + return ExecutionPlan(plan) + def delete(self): """ Deletes graph. diff --git a/tests/functional/test_all.py b/tests/functional/test_all.py index f750955..2fa3222 100644 --- a/tests/functional/test_all.py +++ b/tests/functional/test_all.py @@ -290,6 +290,52 @@ def test_execution_plan(self): redis_graph.delete() + def test_profile(self): + redis_graph = Graph('profile', self.r) + create_query = """CREATE + (:Rider {name:'Valentino Rossi'})-[:rides]->(:Team {name:'Yamaha'}), + (:Rider {name:'Dani Pedrosa'})-[:rides]->(:Team {name:'Honda'}), + (:Rider {name:'Andrea Dovizioso'})-[:rides]->(:Team {name:'Ducati'})""" + redis_graph.query(create_query) + + result = redis_graph.profile("""MATCH (r:Rider)-[:rides]->(t:Team) + WHERE t.name = $name + RETURN r.name, t.name, $params + UNION + MATCH (r:Rider)-[:rides]->(t:Team) + WHERE t.name = $name + RETURN r.name, t.name, $params""", {'name': 'Yehuda'}) + expected = '''\ +Results + Distinct + Join + Project + Conditional Traverse | (t:Team)->(r:Rider) + Filter + Node By Label Scan | (t:Team) + Project + Conditional Traverse | (t:Team)->(r:Rider) + Filter + Node By Label Scan | (t:Team)''' + self.assertEqual(str(result), expected) + + expected = Operation('Results') \ + .append_child(Operation('Distinct') + .append_child(Operation('Join') + .append_child(Operation('Project') + .append_child(Operation('Conditional Traverse', "(t:Team)->(r:Rider)") + .append_child(Operation("Filter") + .append_child(Operation('Node By Label Scan', "(t:Team)"))))) + .append_child(Operation('Project') + .append_child(Operation('Conditional Traverse', "(t:Team)->(r:Rider)") + .append_child(Operation("Filter") + .append_child(Operation('Node By Label Scan', "(t:Team)"))))) + )) + + self.assertEqual(result.structured_plan, expected) + + redis_graph.delete() + def test_query_timeout(self): redis_graph = Graph('timeout', self.r) # Build a sample graph with 1000 nodes. From 9d699376711126ddb1596ea445fb3b22cef5254b Mon Sep 17 00:00:00 2001 From: Avi Avni Date: Mon, 6 Dec 2021 11:03:21 +0200 Subject: [PATCH 2/2] address review --- redisgraph/execution_plan.py | 8 +++- redisgraph/graph.py | 4 +- tests/functional/test_all.py | 90 ++++++++++++++++++++---------------- 3 files changed, 57 insertions(+), 45 deletions(-) diff --git a/redisgraph/execution_plan.py b/redisgraph/execution_plan.py index bea2aec..1b6b9fa 100644 --- a/redisgraph/execution_plan.py +++ b/redisgraph/execution_plan.py @@ -2,6 +2,10 @@ class ProfileStats: + """ + ProfileStats, runtime execution statistics of operation. + """ + def __init__(self, records_produced, execution_time): self.records_produced = records_produced self.execution_time = execution_time @@ -43,7 +47,7 @@ def __eq__(self, o: object) -> bool: return (self.name == o.name and self.args == o.args) def __str__(self) -> str: - args_str = "" if self.args is None else " | " + " | ".join(self.args) + args_str = "" if self.args is None else " | " + self.args return f"{self.name}{args_str}" @@ -151,7 +155,7 @@ def _create_operation(args): execution_time = float(re.search("Execution time: (\\d+.\\d+) ms", args[-1]).group(1)) profile_stats = ProfileStats(records_produced, execution_time) args.pop(-1) - return Operation(name, None if len(args) == 0 else [arg.strip() for arg in args], profile_stats) + return Operation(name, None if len(args) == 0 else args[0].strip(), profile_stats) # iterate plan operations while i < len(self.plan): diff --git a/redisgraph/graph.py b/redisgraph/graph.py index 7dd832a..5bcf669 100644 --- a/redisgraph/graph.py +++ b/redisgraph/graph.py @@ -219,7 +219,7 @@ def query(self, q, params=None, timeout=None, read_only=False): def execution_plan(self, query, params=None): """ Get the execution plan for given query, - GRAPH.EXPLAIN returns an array of operations. + GRAPH.EXPLAIN returns ExecutionPlan object. Args: query: the query that will be executed @@ -234,7 +234,7 @@ def execution_plan(self, query, params=None): def profile(self, query, params=None): """ Get the profield execution plan for given query, - GRAPH.PROFILE returns an array of operations. + GRAPH.PROFILE returns ExecutionPlan object. Args: query: the query that will be executed diff --git a/tests/functional/test_all.py b/tests/functional/test_all.py index 2fa3222..b7c6d2a 100644 --- a/tests/functional/test_all.py +++ b/tests/functional/test_all.py @@ -246,6 +246,7 @@ def test_cached_execution(self): def test_execution_plan(self): redis_graph = Graph('execution_plan', self.r) + # graph creation / population create_query = """CREATE (:Rider {name:'Valentino Rossi'})-[:rides]->(:Team {name:'Yamaha'}), (:Rider {name:'Dani Pedrosa'})-[:rides]->(:Team {name:'Honda'}), @@ -254,11 +255,11 @@ def test_execution_plan(self): result = redis_graph.execution_plan("""MATCH (r:Rider)-[:rides]->(t:Team) WHERE t.name = $name - RETURN r.name, t.name, $params + RETURN r.name, t.name UNION MATCH (r:Rider)-[:rides]->(t:Team) WHERE t.name = $name - RETURN r.name, t.name, $params""", {'name': 'Yehuda'}) + RETURN r.name, t.name""", {'name': 'Yamaha'}) expected = '''\ Results Distinct @@ -292,47 +293,54 @@ def test_execution_plan(self): def test_profile(self): redis_graph = Graph('profile', self.r) - create_query = """CREATE - (:Rider {name:'Valentino Rossi'})-[:rides]->(:Team {name:'Yamaha'}), - (:Rider {name:'Dani Pedrosa'})-[:rides]->(:Team {name:'Honda'}), - (:Rider {name:'Andrea Dovizioso'})-[:rides]->(:Team {name:'Ducati'})""" + # graph creation / population + create_query = """UNWIND range(1, 30) as x CREATE (:Person {id: x})""" redis_graph.query(create_query) - result = redis_graph.profile("""MATCH (r:Rider)-[:rides]->(t:Team) - WHERE t.name = $name - RETURN r.name, t.name, $params - UNION - MATCH (r:Rider)-[:rides]->(t:Team) - WHERE t.name = $name - RETURN r.name, t.name, $params""", {'name': 'Yehuda'}) - expected = '''\ -Results - Distinct - Join - Project - Conditional Traverse | (t:Team)->(r:Rider) - Filter - Node By Label Scan | (t:Team) - Project - Conditional Traverse | (t:Team)->(r:Rider) - Filter - Node By Label Scan | (t:Team)''' - self.assertEqual(str(result), expected) - - expected = Operation('Results') \ - .append_child(Operation('Distinct') - .append_child(Operation('Join') - .append_child(Operation('Project') - .append_child(Operation('Conditional Traverse', "(t:Team)->(r:Rider)") - .append_child(Operation("Filter") - .append_child(Operation('Node By Label Scan', "(t:Team)"))))) - .append_child(Operation('Project') - .append_child(Operation('Conditional Traverse', "(t:Team)->(r:Rider)") - .append_child(Operation("Filter") - .append_child(Operation('Node By Label Scan', "(t:Team)"))))) - )) - - self.assertEqual(result.structured_plan, expected) + plan = redis_graph.profile("""MATCH (p:Person) + WHERE p.id > 15 + RETURN p""") + + results = plan.structured_plan + self.assertEqual(results.name, "Results") + self.assertEqual(results.profile_stats.records_produced, 15) + self.assertGreater(results.profile_stats.execution_time, 0) + + project = results.children[0] + self.assertEqual(project.name, "Project") + self.assertEqual(project.profile_stats.records_produced, 15) + self.assertGreater(project.profile_stats.execution_time, 0) + + filter = project.children[0] + self.assertEqual(filter.name, "Filter") + self.assertEqual(filter.profile_stats.records_produced, 15) + self.assertGreater(filter.profile_stats.execution_time, 0) + + node_by_label_scan = filter.children[0] + self.assertEqual(node_by_label_scan.name, "Node By Label Scan") + self.assertEqual(node_by_label_scan.profile_stats.records_produced, 30) + self.assertGreater(node_by_label_scan.profile_stats.execution_time, 0) + + redis_graph.query("CREATE INDEX FOR (p:Person) ON (p.id)") + + plan = redis_graph.profile("""MATCH (p:Person) + WHERE p.id > 15 + RETURN p""") + + results = plan.structured_plan + self.assertEqual(results.name, "Results") + self.assertEqual(results.profile_stats.records_produced, 15) + self.assertGreater(results.profile_stats.execution_time, 0) + + project = results.children[0] + self.assertEqual(project.name, "Project") + self.assertEqual(project.profile_stats.records_produced, 15) + self.assertGreater(project.profile_stats.execution_time, 0) + + node_by_index_scan = project.children[0] + self.assertEqual(node_by_index_scan.name, "Node By Index Scan") + self.assertEqual(node_by_index_scan.profile_stats.records_produced, 15) + self.assertGreater(node_by_index_scan.profile_stats.execution_time, 0) redis_graph.delete()