Add past CDO ontology and version IRIs to existing-concept review

A follow-on patch will regenerate Make-managed files.

References:
* #40

Signed-off-by: Alex Nelson <alexander.nelson@nist.gov>

Author: ajnelson-nist

CONTRIBUTE.md

@@ -27,10 +27,12 @@ pushd case_utils/ontology
   git add case-0.6.0.ttl  # Assuming CASE 0.6.0 was just released.
   # and/or
   git add uco-0.8.0.ttl   # Assuming UCO 0.8.0 was adopted in CASE 0.6.0.
+
+  git add ontology_and_version_iris.txt
 popd
 make check
 # Assuming `make check` passes:
-git commit -m "Build CASE 0.6.0 monolithic .ttl files" case_utils/ontology/case-0.6.0-subclasses.ttl case_utils/ontology/case-0.6.0.ttl
+git commit -m "Build CASE 0.6.0 monolithic .ttl files" case_utils/ontology/case-0.6.0-subclasses.ttl case_utils/ontology/case-0.6.0.ttl case_utils/ontology/ontology_and_version_iris.txt
 git commit -m "Update CASE ontology pointer to version 0.6.0" dependencies/CASE case_utils/ontology/version_info.py
 ```
 
@@ -43,4 +45,4 @@ pre-commit --version
 The `pre-commit` tool hooks into Git's commit machinery to run a set of linters and static analyzers over each change. To install `pre-commit` into Git's hooks, run:
 ```bash
 pre-commit install
-```
+```

case_utils/case_validate/__init__.py

@@ -180,7 +180,9 @@ def main() -> None:
             _logger.debug("arg_ontology_graph = %r.", arg_ontology_graph)
             ontology_graph.parse(arg_ontology_graph)
 
+    # Construct set of CDO concepts for data graph concept-existence review.
     cdo_concepts: typing.Set[rdflib.URIRef] = set()
+
     for n_structural_class in [
         NS_OWL.Class,
         NS_OWL.AnnotationProperty,
@@ -214,6 +216,16 @@ def main() -> None:
             continue
         cdo_concepts.add(ontology_triple[0])
 
+    # Also load historical ontology and version IRIs.
+    ontology_and_version_iris_data = importlib.resources.read_text(
+        case_utils.ontology, "ontology_and_version_iris.txt"
+    )
+    for line in ontology_and_version_iris_data.split("\n"):
+        cleaned_line = line.strip()
+        if cleaned_line == "":
+            continue
+        cdo_concepts.add(rdflib.URIRef(cleaned_line))
+
     data_cdo_concepts: typing.Set[rdflib.URIRef] = set()
     for data_triple in data_graph.triples((None, None, None)):
         for data_triple_member in data_triple:

case_utils/ontology/Makefile

@@ -24,7 +24,7 @@ RDF_TOOLKIT_JAR := $(uco_srcdir)/lib/rdf-toolkit.jar
 case_version := $(shell python3 version_info.py)
 
 all: \
-  case-$(case_version)-subclasses.ttl
+  ontology_and_version_iris.txt
 
 .PRECIOUS: \
   case-$(case_version).ttl
@@ -79,3 +79,16 @@ case-$(case_version)-subclasses.ttl: \
 clean:
 	@rm -f \
 	  case-$(case_version)*.ttl
+
+ontology_and_version_iris.txt: \
+  src/ontology_and_version_iris.py \
+  case-$(case_version)-subclasses.ttl
+	# Guarantee venv is built.  (Same rationale as in the subclasses.ttl recipe.)
+	$(MAKE) \
+	  --directory $(case_srcdir)/tests \
+	    .venv.done.log
+	source $(case_srcdir)/tests/venv/bin/activate \
+	  && python3 src/ontology_and_version_iris.py \
+	    _$@ \
+	    case-*.ttl
+	mv _$@ $@

case_utils/ontology/src/ontology_and_version_iris.py

@@ -0,0 +1,90 @@
+#!/usr/bin/env python3
+
+# This software was developed at the National Institute of Standards
+# and Technology by employees of the Federal Government in the course
+# of their official duties. Pursuant to title 17 Section 105 of the
+# United States Code this software is not subject to copyright
+# protection and is in the public domain. NIST assumes no
+# responsibility whatsoever for its use by other parties, and makes
+# no guarantees, expressed or implied, about its quality,
+# reliability, or any other characteristic.
+#
+# We would appreciate acknowledgement if the software is used.
+
+"""
+This script creates a list of all ontology and version IRIs that have ever existed in a CDO ontology to describe a CDO ontology.  I.e. the subject of triples with owl:Ontology as predicate are included, as are the objects of version-referencing triples (owl:versionIRI, owl:incompatibleWith, etc.).
+"""
+
+__version__ = "0.1.0"
+
+import argparse
+import typing
+
+import rdflib
+
+NS_OWL = rdflib.OWL
+NS_RDF = rdflib.RDF
+
+
+def concept_is_cdo_concept(n_concept: rdflib.URIRef) -> bool:
+    """
+    This function is purposefully distinct from the function used in case_validate.  Within this script, the publishing history of CASE and UCO is reviewed."""
+    concept_iri = str(n_concept)
+    return (
+        concept_iri.startswith("https://ontology.unifiedcyberontology.org/")
+        or concept_iri.startswith("https://ontology.caseontology.org/")
+        or concept_iri.startswith("https://unifiedcyberontology.org/ontology/")
+        or concept_iri.startswith("https://caseontology.org/ontology/")
+        or concept_iri == "http://case.example.org/core"
+    )
+
+
+def extract_ontology_iris(ontology_graph: rdflib.Graph) -> typing.Set[rdflib.URIRef]:
+    """
+    Return all concepts describing the OWL Ontology in the input graph.  This does not return classes, properties, etc. defined within the ontology; instead, it only returns the ontology IRI and annotations about the ontology.
+    """
+    ontology_concepts: typing.Set[rdflib.URIRef] = set()
+    for n_ontology_predicate in [
+        NS_OWL.backwardCompatibleWith,
+        NS_OWL.imports,
+        NS_OWL.incompatibleWith,
+        NS_OWL.priorVersion,
+        NS_OWL.versionIRI,
+    ]:
+        for ontology_triple in ontology_graph.triples(
+            (None, n_ontology_predicate, None)
+        ):
+            assert isinstance(ontology_triple[0], rdflib.URIRef)
+            assert isinstance(ontology_triple[2], rdflib.URIRef)
+            ontology_concepts.add(ontology_triple[0])
+            ontology_concepts.add(ontology_triple[2])
+    for ontology_triple in ontology_graph.triples((None, NS_RDF.type, NS_OWL.Ontology)):
+        if not isinstance(ontology_triple[0], rdflib.URIRef):
+            continue
+        if concept_is_cdo_concept(ontology_triple[0]):
+            ontology_concepts.add(ontology_triple[0])
+    return ontology_concepts
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("out_txt")
+    parser.add_argument("in_ttl", nargs="+")
+    args = parser.parse_args()
+
+    cdo_concepts: typing.Set[rdflib.URIRef] = set()
+    for in_ttl in args.in_ttl:
+        ontology_graph = rdflib.Graph()
+        ontology_graph.parse(in_ttl)
+        ontology_concepts = extract_ontology_iris(ontology_graph)
+        for ontology_concept in ontology_concepts:
+            if concept_is_cdo_concept(ontology_concept):
+                cdo_concepts.add(ontology_concept)
+
+    with open(args.out_txt, "w") as out_fh:
+        for cdo_concept in sorted(cdo_concepts):
+            print(cdo_concept, file=out_fh)
+
+
+if __name__ == "__main__":
+    main()

tests/case_utils/case_validate/cli/Makefile

@@ -42,6 +42,8 @@ files_to_generate := \
   format_unspecified_output_turtle.ttl \
   format_unspecified_output_txt.txt \
   format_unspecified_output_unspecified.txt \
+  past_version_reference_PASS.txt \
+  past_version_reference_XFAIL.txt \
   split_data_graph_PASS.txt \
   split_data_graph_XFAIL.txt \
   thing_metashacl_PASS.txt
@@ -220,6 +222,36 @@ format_unspecified_output_unspecified.txt: \
 	    > _$@
 	mv _$@ $@
 
+past_version_reference_PASS.txt: \
+  $(tests_srcdir)/.venv.done.log \
+  $(top_srcdir)/.ontology.done.log \
+  $(top_srcdir)/case_utils/case_validate/__init__.py \
+  $(top_srcdir)/case_utils/ontology/__init__.py \
+  $(top_srcdir)/case_utils/ontology/ontology_and_version_iris.txt \
+  past_version_reference_PASS.ttl
+	rm -f _$@
+	source $(tests_srcdir)/venv/bin/activate \
+	  && case_validate \
+	    past_version_reference_PASS.ttl \
+	    > _$@
+	mv _$@ $@
+
+past_version_reference_XFAIL.txt: \
+  $(tests_srcdir)/.venv.done.log \
+  $(top_srcdir)/.ontology.done.log \
+  $(top_srcdir)/case_utils/case_validate/__init__.py \
+  $(top_srcdir)/case_utils/ontology/__init__.py \
+  $(top_srcdir)/case_utils/ontology/ontology_and_version_iris.txt \
+  past_version_reference_XFAIL.ttl
+	rm -f _$@
+	source $(tests_srcdir)/venv/bin/activate \
+	  && case_validate \
+	    past_version_reference_XFAIL.ttl \
+	    > _$@ \
+	    ; rc=$$? ; test 1 -eq $$rc
+	test -s _$@
+	mv _$@ $@
+
 split_data_graph_PASS.txt: \
   $(tests_srcdir)/.venv.done.log \
   $(top_srcdir)/.ontology.done.log \

tests/case_utils/case_validate/cli/past_version_reference_PASS.ttl

@@ -0,0 +1,12 @@
+@prefix owl: <http://www.w3.org/2002/07/owl#> .
+@prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> .
+@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> .
+@prefix uco-core: <https://ontology.unifiedcyberontology.org/uco/core/> .
+@prefix xsd: <http://www.w3.org/2001/XMLSchema#> .
+
+<urn:example:thing-a5ba4c5b-9e12-41e8-b05d-4dfb0dd61ec3>
+	a uco-core:UcoThing ;
+	rdfs:comment "This node maintains a historic reference to 0.9.1 of the UCO namespace.  This must not trigger an undefined-concept warning."@en ;
+	rdfs:seeAlso uco-core:0.9.1 ;
+	.
+

tests/case_utils/case_validate/cli/past_version_reference_XFAIL.ttl

@@ -0,0 +1,12 @@
+@prefix owl: <http://www.w3.org/2002/07/owl#> .
+@prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> .
+@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> .
+@prefix uco-core: <https://ontology.unifiedcyberontology.org/uco/core/> .
+@prefix xsd: <http://www.w3.org/2001/XMLSchema#> .
+
+<urn:example:thing-bcd908d5-d89b-4352-afe0-190ce331fd5b>
+	a uco-core:UcoThing ;
+	rdfs:comment "This node references a never-published UCO Core namespace version.  This is expected to trigger a warning."@en ;
+	rdfs:seeAlso uco-core:0.0.1 ;
+	.
+