add config options for narrow deep paths mutation

wip: narrow deep paths now creates a chain of var edges (test included)

fix logging config bug on windows systems

added to_find_edge_var_for_narrow_path_query

Author: nandikaK

config/defaults.py

@@ -78,6 +78,9 @@
 MUTPB_FV_SAMPLE_MAXN = 32  # max n of instantiations to sample from top k
 MUTPB_FV_QUERY_LIMIT = 256  # SPARQL query limit for the top k instantiations
 MUTPB_SP = 0.05  # prob to simplify pattern (warning: can restrict exploration)
-
+MUTPB_DN = 0.05  # prob to try a deep and narrow paths mutation
+MUTPB_DN_MIN_LEN = 2  # minimum length of the deep and narrow paths
+MUTPB_DN_MAX_LEN = 10  # absolute max of path length if not stopped by term_pb
+MUTPB_DN_TERM_PB = 0.3  # prob to terminate node expansion each step > min_len
 # for import in helpers and __init__
 __all__ = [_v for _v in globals().keys() if _v.isupper()]

gp_learner.py

@@ -413,19 +413,53 @@ def mutate_del_triple(child):
         return new_child
 
 
-def mutate_expand_node(child, pb_en_out_link):
-    # TODO: can maybe be improved by sparqling
-    nodes = list(child.nodes)
-    node = random.choice(nodes)
+def _mutate_expand_node_helper(node, pb_en_out_link=config.MUTPB_EN_OUT_LINK):
     var_edge = gen_random_var()
     var_node = gen_random_var()
     if random.random() < pb_en_out_link:
         new_triple = (node, var_edge, var_node)
     else:
         new_triple = (var_node, var_edge, node)
+    return new_triple, var_node
+
+
+def mutate_expand_node(child, node=None):
+    # TODO: can maybe be improved by sparqling
+    if not node:
+        nodes = list(child.nodes)
+        node = random.choice(nodes)
+    new_triple, _ = _mutate_expand_node_helper(node)
     return child + (new_triple,)
 
 
+def mutate_deep_narrow_path(
+        child,
+        min_len=config.MUTPB_DN_MIN_LEN,
+        max_len=config.MUTPB_DN_MAX_LEN,
+        term_pb=config.MUTPB_DN_TERM_PB,
+        pb_en_out_link=config.MUTPB_EN_OUT_LINK,
+):
+    assert isinstance(child, GraphPattern)
+    nodes = list(child.nodes)
+    start_node = random.choice(nodes)
+    # target_nodes = set(nodes) - {start_node}
+    gp = child
+    hop = 0
+    while True:
+        if hop >= min_len and random.random() < term_pb:
+            break
+        if hop >= max_len:
+            break
+        hop += 1
+        new_triple, var_node = _mutate_expand_node_helper(start_node)
+        gp += [new_triple]
+        start_node = var_node
+
+    # TODO: insert connection to a target node
+    # TODO: fix edge or node ( to_count_var_over_values_query)
+    return gp
+
+
 def mutate_add_edge(child):
     # TODO: can maybe be improved by sparqling
     nodes = list(child.nodes)
@@ -647,7 +681,6 @@ def mutate(
         pb_ae=config.MUTPB_AE,
         pb_dt=config.MUTPB_DT,
         pb_en=config.MUTPB_EN,
-        pb_en_out_link=config.MUTPB_EN_OUT_LINK,
         pb_fv=config.MUTPB_FV,
         pb_id=config.MUTPB_ID,
         pb_iv=config.MUTPB_IV,
@@ -678,7 +711,7 @@ def mutate(
         child = mutate_del_triple(child)
 
     if random.random() < pb_en:
-        child = mutate_expand_node(child, pb_en_out_link)
+        child = mutate_expand_node(child)
     if random.random() < pb_ae:
         child = mutate_add_edge(child)
 
@@ -694,7 +727,6 @@ def mutate(
     else:
         children = [child]
 
-
     # TODO: deep & narrow paths mutation
 
     children = {

graph_pattern.py

@@ -41,8 +41,10 @@
 TARGET_VAR = Variable('target')
 ASK_VAR = Variable('ask')
 COUNT_VAR = Variable('count')
-
-
+EDGE_VAR_COUNT = Variable('edge_count_var')
+NODE_VAR_COUNT = Variable('node_count_var')
+MAX_NODE_COUNT = Variable('maximum node count')
+PRIO_VAR = Variable('priority')
 def gen_random_var():
     return Variable(RANDOM_VAR_PREFIX + ''.join(
         random.choice(string.ascii_letters + string.digits)
@@ -709,6 +711,90 @@ def to_count_var_over_values_query(self, var, vars_, values, limit):
         res += 'LIMIT %d\n' % limit
         return self._sparql_prefix(res)
 
+    def to_find_edge_var_for_narrow_path_query(self, edge_var, node_var,
+                                               vars_, filter_node_count,
+                                               filter_edge_count, limit_res):
+        """Counts possible substitutions for edge_var to get a narrow path
+
+        Meant to perform a query like this:
+        SELECT *
+        {
+          {
+            SELECT
+              ?edge_var
+              (COUNT(*) AS ?edge_var_count)
+              (MAX(?node_var_count) AS ?max_node_count)
+              (COUNT(*)/AVG(?node_var_count) as ?prio_var)
+            {
+              SELECT DISTINCT
+                ?source ?target ?edge_var (COUNT(?node_var) AS ?node_var_count)
+              {
+                VALUES (?source ?target) {
+                  (dbr:Adolescence dbr:Youth)
+                  (dbr:Adult dbr:Child)
+                  (dbr:Angel dbr:Heaven)
+                  (dbr:Arithmetic dbr:Mathematics)
+                }
+                ?node_var ?edge_var ?source .
+                ?source dbo:wikiPageWikiLink ?target .
+              }
+            }
+            GROUP BY ?edge_var
+            ORDER BY DESC(?edge_var_count)
+          }
+          FILTER(?max_node_count < 10 && ?edge_var_count > 1)
+        }
+        ORDER BY DESC(?prio_var)
+        LIMIT 32
+
+        :param edge_var: Edge variable to find substitution for.
+        :param node_var: Node variable to count.
+        :param vars_: List of vars to fix values for (e.g. ?source, ?target).
+        :param values: List of value lists for vars_.
+        :param filter_node_count: Filter on node count of edge variable.
+        :param filter_edge_count: Filter for edge count of triples.
+        :param limit_res : limit result size
+        :return: Query String.
+        """
+
+        res = 'SELECT * WHERE {\n'
+        res += ' {\n'\
+               '  SELECT %s (COUNT(*) as %s) (Max(%s) AS %s) ' \
+               '   (COUNT(*)/AVG(%s) AS %s) WHERE {\n' % (
+                     edge_var.n3(), EDGE_VAR_COUNT.n3(),
+                     NODE_VAR_COUNT.n3(), MAX_NODE_COUNT.n3(),
+                     NODE_VAR_COUNT.n3(), PRIO_VAR.n3())
+        res += '    SELECT DISTINCT %s %s (COUNT(%s) AS %s) WHERE {\n' % (
+                   ' '.join([v.n3() for v in vars_]),
+                   edge_var.n3(), node_var.n3(), NODE_VAR_COUNT.n3())
+        # res += self._sparql_values_part(values)
+        res += 'VALUES(%s) {\n' \
+               '(dbr: Adolescence dbr: Youth)' \
+               '(dbr:Adult dbr:Child)' \
+               '(dbr:Angel  dbr:Heaven)' \
+               '(dbr:Arithmetic dbr:Mathematics)' \
+               '}\n' % (' '.join([v.n3() for v in vars_]))
+        # triples part
+        tres = []
+        for s, p, o in self:
+            tres.append('%s %s %s .' % (s.n3(), p.n3(), o.n3()))
+        indent = ' ' * 3
+        triples = indent + ('\n' + indent).join(tres) + '\n'
+        res += triples
+        res += '    }\n'\
+               '   }\n'
+        res += '   GROUP BY %s\n' % edge_var.n3()
+        res += '   ORDER BY DESC(%s)\n' % EDGE_VAR_COUNT.n3()
+        res += '  }\n'
+        res += '  FILTER(%s < %d && %s > %d)\n' % (MAX_NODE_COUNT.n3(),
+                                                 filter_node_count,
+                                                 EDGE_VAR_COUNT.n3(),
+                                                 filter_edge_count)
+        res += '}\n'
+        res += 'ORDER BY DESC(%s)\n' % PRIO_VAR.n3()
+        res += 'LIMIT %d' % limit_res
+        return self._sparql_prefix(res)
+
     def to_dict(self):
         return {
             'fitness': self.fitness.values if self.fitness.valid else (),

logging_config.py

@@ -191,7 +191,7 @@ def doRollover(self):
     # 'loggers': {}  # all other loggers except for root
 }
 
-if os.uname()[0] == 'Darwin':
+if os.name == 'posix' and os.uname()[0] == 'Darwin':
     logging_config['formatters']['tts_formatter'] = {
         'format': format_str_tts
     }

requirements.txt

@@ -5,11 +5,11 @@ matplotlib==1.5.1
 networkx==1.11
 nose==1.3.7
 numpy==1.11.0
-pygraphviz==1.3.1
+# pygraphviz==1.3.1
 requests==2.9.1
 rdflib==4.2.1
 scikit-learn==0.17.1
-scipy==0.17.0
+scipy
 scoop==0.7.1.1
 six==1.10.0
 SPARQLWrapper==1.7.6

tests/test_gp_learner_offline.py

@@ -13,6 +13,7 @@
 from gp_learner import mutate_increase_dist
 from gp_learner import mutate_merge_var
 from gp_learner import mutate_simplify_pattern
+from gp_learner import mutate_deep_narrow_path
 from graph_pattern import GraphPattern
 from graph_pattern import SOURCE_VAR
 from graph_pattern import TARGET_VAR
@@ -108,6 +109,35 @@ def test_mutate_merge_var():
         assert False, "merge never reached one of the cases: %s" % cases
 
 
+def test_mutate_deep_narrow_path():
+    p = Variable('p')
+    gp = GraphPattern([
+        (SOURCE_VAR, p, TARGET_VAR)
+    ])
+    child = mutate_deep_narrow_path(gp)
+    assert gp == child or len(child) > len(gp)
+    print(gp)
+    print(child)
+
+
+def test_to_find_edge_var_for_narrow_path_query():
+    node_var = Variable('node_variable')
+    edge_var = Variable('edge_variable')
+    gp = GraphPattern([
+        (node_var, edge_var, SOURCE_VAR),
+        (SOURCE_VAR, wikilink, TARGET_VAR)
+    ])
+    filter_node_count = 10
+    filter_edge_count = 1
+    limit_res = 32
+    vars_ = {SOURCE_VAR,TARGET_VAR}
+    res = GraphPattern.to_find_edge_var_for_narrow_path_query(gp, edge_var, node_var,
+                                           vars_, filter_node_count,
+                                           filter_edge_count, limit_res)
+    print(gp)
+    print(res)
+
+
 def test_simplify_pattern():
     gp = GraphPattern([(SOURCE_VAR, wikilink, TARGET_VAR)])
     res = mutate_simplify_pattern(gp)
@@ -271,3 +301,8 @@ def test_remaining_gain_sample_gtps():
 
 def test_gtp_scores():
     assert gtp_scores - gtp_scores == 0
+
+
+if __name__ == '__main__':
+    # test_mutate_deep_narrow_path()
+    test_to_find_edge_var_for_narrow_path_query()