Consolidate reviews

This patch consolidates the review of CASE-Corpora's knowledge base
graph against prerelease states of CASE and UCO.  The current
implementation reviews each dataset against CASE's `develop` and
`unstable` states.  Before resolution of the slowdown noted in
`case-utils` PR 72, this is a significant time cost that can be
decreased by instead reviewing the consolidated knowledge base graph at
`/catalog/kb-all.ttl`.  This will be a more significant time cost if
CASE-Corpora starts reviewing against the `-2.0.0` prerelease builds,
using the review scheme outlined in UCO Issue 493.

This patch also speeds up CI's review of each dataset by only running
per-dataset validation against the current CASE release when not
operating in a GitHub Action.  This supports local (offline) development
still running review in a focused dataset directory.  (The Bash syntax
used for variable definition detection is compatible with Bash's
`set -u`.  See tldp.org reference.)

A follow-on patch will regenerate Make-managed files.

References:
* casework/CASE-Utilities-Python#72
* ucoProject/UCO#493
* https://tldp.org/LDP/abs/html/parameter-substitution.html

Signed-off-by: Alex Nelson <alexander.nelson@nist.gov>

Author: ajnelson-nist

catalog/.gitignore

@@ -1,4 +1,4 @@
+*validation*.ttl
 datasets.md
 devices.md
 kb-*.ttl
-validate-*.ttl

catalog/Makefile

@@ -22,7 +22,7 @@ all:
 	  --file postvisit.mk
 
 check: \
-  validate-catalog.ttl \
+  catalog_validation.ttl \
   all
 	$(MAKE) \
 	  --directory datasets \
@@ -33,10 +33,10 @@ check: \
 
 clean:
 	@rm -f \
-	  kb-*.ttl \
-	  validate-*.ttl
+	  *_validation*.ttl \
+	  kb-*.ttl
 
-validate-catalog.ttl: \
+catalog_validation.ttl: \
   $(top_srcdir)/.venv.done.log \
   $(top_srcdir)/dependencies/dependencies.ttl \
   $(top_srcdir)/ontology/case-corpora.ttl \

catalog/postvisit.mk

@@ -29,14 +29,15 @@ kb_datasets_dependencies := \
   $(wildcard datasets/*/distribution.ttl)
 
 kb_all_dependencies := \
-  kb-datasets.ttl \
   $(top_srcdir)/taxonomy/devices/drafting.ttl \
+  kb-datasets.ttl \
   $(wildcard datasets/*/generated-*.ttl) \
   $(wildcard datasets/*/supplemental.ttl)
 
 all: \
   README.md \
-  kb-all.ttl
+  kb-all_validation-develop.ttl \
+  kb-all_validation-unstable.ttl
 
 README.md: \
   README.md.in \
@@ -49,7 +50,7 @@ README.md: \
 	mv _$@ $@
 
 check: \
-  validate-kb-all.ttl
+  kb-all_validation.ttl
 
 datasets.md: \
   $(top_srcdir)/ontology/case-corpora.ttl \
@@ -82,16 +83,22 @@ kb-all.ttl: \
 	rm __$@
 	mv _$@ $@
 
-kb-datasets.ttl: \
-  $(kb_datasets_dependencies) \
-  $(rdf_toolkit_jar) \
-  $(top_srcdir)/.venv.done.log
+kb-all_validation.ttl: \
+  kb-all.ttl \
+  kb-datasets_validation.ttl
 	rm -f __$@ _$@
 	source $(top_srcdir)/venv/bin/activate \
-	  && rdfpipe \
-	    --output-format turtle \
-	    $(kb_datasets_dependencies) \
-	    > __$@
+	  && case_validate \
+	    --allow-infos \
+	    --format turtle \
+	    --inference rdfs \
+	    --ontology-graph $(top_srcdir)/dependencies/dependencies.ttl \
+	    --ontology-graph $(top_srcdir)/ontology/case-corpora.ttl \
+	    --ontology-graph $(top_srcdir)/shapes/debug.ttl \
+	    --ontology-graph $(top_srcdir)/shapes/shapes.ttl \
+	    --output __$@ \
+	    kb-all.ttl \
+	    || (cat __$@ ; exit 1)
 	java -jar $(rdf_toolkit_jar) \
 	  --inline-blank-nodes \
 	  --source-format turtle \
@@ -101,28 +108,82 @@ kb-datasets.ttl: \
 	rm __$@
 	mv _$@ $@
 
-validate-kb-all.ttl: \
-  kb-all.ttl \
-  validate-kb-datasets.ttl
-	rm -f _$@
+kb-all_validation-develop.ttl: \
+  $(top_srcdir)/dependencies/CASE-develop.ttl \
+  kb-all.ttl
+	rm -f __$@ _$@
 	source $(top_srcdir)/venv/bin/activate \
 	  && case_validate \
 	    --allow-infos \
+	    --built-version none \
 	    --format turtle \
 	    --inference rdfs \
+	    --ontology-graph $(top_srcdir)/dependencies/CASE-develop.ttl \
 	    --ontology-graph $(top_srcdir)/dependencies/dependencies.ttl \
 	    --ontology-graph $(top_srcdir)/ontology/case-corpora.ttl \
-	    --ontology-graph $(top_srcdir)/shapes/debug.ttl \
 	    --ontology-graph $(top_srcdir)/shapes/shapes.ttl \
-	    --output _$@ \
-	    kb-all.ttl \
-	    || (cat _$@ ; exit 1)
+	    --output __$@ \
+	    kb-all.ttl
+	java -jar $(rdf_toolkit_jar) \
+	  --inline-blank-nodes \
+	  --source-format turtle \
+	  --source __$@ \
+	  --target-format turtle \
+	  --target _$@
+	rm __$@
 	mv _$@ $@
 
-validate-kb-datasets.ttl: \
-  kb-datasets.ttl \
-  validate-catalog.ttl
-	rm -f _$@
+kb-all_validation-unstable.ttl: \
+  $(top_srcdir)/dependencies/CASE-unstable.ttl \
+  kb-all.ttl
+	rm -f __$@ _$@
+	source $(top_srcdir)/venv/bin/activate \
+	  && case_validate \
+	    --allow-infos \
+	    --built-version none \
+	    --format turtle \
+	    --inference rdfs \
+	    --ontology-graph $(top_srcdir)/dependencies/CASE-unstable.ttl \
+	    --ontology-graph $(top_srcdir)/dependencies/dependencies.ttl \
+	    --ontology-graph $(top_srcdir)/ontology/case-corpora.ttl \
+	    --ontology-graph $(top_srcdir)/shapes/shapes.ttl \
+	    --output __$@ \
+	    kb-all.ttl
+	java -jar $(rdf_toolkit_jar) \
+	  --inline-blank-nodes \
+	  --source-format turtle \
+	  --source __$@ \
+	  --target-format turtle \
+	  --target _$@
+	rm __$@
+	mv _$@ $@
+
+kb-datasets.ttl: \
+  $(kb_datasets_dependencies) \
+  $(rdf_toolkit_jar) \
+  $(top_srcdir)/dependencies/dependencies.ttl \
+  $(top_srcdir)/ontology/case-corpora.ttl \
+  $(top_srcdir)/shapes/shapes.ttl \
+  $(top_srcdir)/.venv.done.log
+	rm -f __$@ _$@
+	source $(top_srcdir)/venv/bin/activate \
+	  && rdfpipe \
+	    --output-format turtle \
+	    $(kb_datasets_dependencies) \
+	    > __$@
+	java -jar $(rdf_toolkit_jar) \
+	  --inline-blank-nodes \
+	  --source-format turtle \
+	  --source __$@ \
+	  --target-format turtle \
+	  --target _$@
+	rm __$@
+	mv _$@ $@
+
+kb-datasets_validation.ttl: \
+  catalog_validation.ttl \
+  kb-datasets.ttl
+	rm -f __$@ _$@
 	source $(top_srcdir)/venv/bin/activate \
 	  && case_validate \
 	    --allow-infos \
@@ -132,7 +193,14 @@ validate-kb-datasets.ttl: \
 	    --ontology-graph $(top_srcdir)/ontology/case-corpora.ttl \
 	    --ontology-graph $(top_srcdir)/shapes/debug.ttl \
 	    --ontology-graph $(top_srcdir)/shapes/shapes.ttl \
-	    --output _$@ \
+	    --output __$@ \
 	    kb-datasets.ttl \
-	    || (cat _$@ ; exit 1)
+	    || (cat __$@ ; exit 1)
+	java -jar $(rdf_toolkit_jar) \
+	  --inline-blank-nodes \
+	  --source-format turtle \
+	  --source __$@ \
+	  --target-format turtle \
+	  --target _$@
+	rm __$@
 	mv _$@ $@

src/dataset.mk

@@ -24,8 +24,7 @@ maybe_ground_truth_graph := $(wildcard ground-truth.*)
 rdf_toolkit_jar := $(top_srcdir)/dependencies/CASE/dependencies/UCO/lib/rdf-toolkit.jar
 
 all: \
-  kb_validation-CASE-develop.ttl \
-  kb_validation-CASE-unstable.ttl
+  kb.ttl
 
 .PHONY: \
   check-pytest \
@@ -52,16 +51,20 @@ kb.ttl: \
 	    $(maybe_ground_truth_graph) \
 	    generated-*.ttl \
 	    > __$@
-	source $(top_srcdir)/venv/bin/activate \
-	  && case_validate \
-	    --allow-infos \
-	    --inference rdfs \
-	    --ontology-graph $(top_srcdir)/dependencies/dependencies.ttl \
-	    --ontology-graph $(top_srcdir)/ontology/case-corpora.ttl \
-	    --ontology-graph $(top_srcdir)/shapes/debug.ttl \
-	    --ontology-graph $(top_srcdir)/shapes/shapes.ttl \
-	    --ontology-graph $(top_srcdir)/taxonomy/devices/drafting.ttl \
-	    __$@
+	# Skip per-dataset validation when this file is called in a GitHub action.
+	test ! -z "$${GITHUB_ACTIONS+x}" \
+	  || ( \
+	    source $(top_srcdir)/venv/bin/activate \
+	      && case_validate \
+	        --allow-infos \
+	        --inference rdfs \
+	        --ontology-graph $(top_srcdir)/dependencies/dependencies.ttl \
+	        --ontology-graph $(top_srcdir)/ontology/case-corpora.ttl \
+	        --ontology-graph $(top_srcdir)/shapes/debug.ttl \
+	        --ontology-graph $(top_srcdir)/shapes/shapes.ttl \
+	        --ontology-graph $(top_srcdir)/taxonomy/devices/drafting.ttl \
+	        __$@ \
+	    )
 	java -jar $(rdf_toolkit_jar) \
 	  --inline-blank-nodes \
 	  --source-format turtle \
@@ -129,57 +132,3 @@ generated-prov.ttl: \
 	  --target _$@
 	rm __$@
 	mv _$@ $@
-
-kb_validation-CASE-develop.ttl: \
-  $(top_srcdir)/dependencies/CASE-develop.ttl \
-  $(top_srcdir)/taxonomy/devices/drafting.ttl \
-  kb.ttl
-	rm -f __$@ _$@
-	source $(top_srcdir)/venv/bin/activate \
-	  && case_validate \
-	    --allow-infos \
-	    --built-version none \
-	    --format turtle \
-	    --inference rdfs \
-	    --ontology-graph $(top_srcdir)/dependencies/CASE-develop.ttl \
-	    --ontology-graph $(top_srcdir)/dependencies/dependencies.ttl \
-	    --ontology-graph $(top_srcdir)/ontology/case-corpora.ttl \
-	    --ontology-graph $(top_srcdir)/shapes/shapes.ttl \
-	    --ontology-graph $(top_srcdir)/taxonomy/devices/drafting.ttl \
-	    --output __$@ \
-	    kb.ttl
-	java -jar $(rdf_toolkit_jar) \
-	  --inline-blank-nodes \
-	  --source-format turtle \
-	  --source __$@ \
-	  --target-format turtle \
-	  --target _$@
-	rm __$@
-	mv _$@ $@
-
-kb_validation-CASE-unstable.ttl: \
-  $(top_srcdir)/dependencies/CASE-unstable.ttl \
-  $(top_srcdir)/taxonomy/devices/drafting.ttl \
-  kb.ttl
-	rm -f __$@ _$@
-	source $(top_srcdir)/venv/bin/activate \
-	  && case_validate \
-	    --allow-infos \
-	    --built-version none \
-	    --format turtle \
-	    --inference rdfs \
-	    --ontology-graph $(top_srcdir)/dependencies/CASE-unstable.ttl \
-	    --ontology-graph $(top_srcdir)/dependencies/dependencies.ttl \
-	    --ontology-graph $(top_srcdir)/ontology/case-corpora.ttl \
-	    --ontology-graph $(top_srcdir)/shapes/shapes.ttl \
-	    --ontology-graph $(top_srcdir)/taxonomy/devices/drafting.ttl \
-	    --output __$@ \
-	    kb.ttl
-	java -jar $(rdf_toolkit_jar) \
-	  --inline-blank-nodes \
-	  --source-format turtle \
-	  --source __$@ \
-	  --target-format turtle \
-	  --target _$@
-	rm __$@
-	mv _$@ $@