Make large, recursive schemas diff-able by deferring computation of diffs

cli/src/main/java/org/openapitools/openapidiff/cli/Main.java

@@ -1,14 +1,11 @@
 package org.openapitools.openapidiff.cli;
 
+import io.swagger.v3.parser.core.models.AuthorizationValue;
 import java.io.File;
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
-import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
-import java.util.stream.Collectors;
-
-import io.swagger.v3.parser.core.models.AuthorizationValue;
 import org.apache.commons.cli.CommandLine;
 import org.apache.commons.cli.CommandLineParser;
 import org.apache.commons.cli.DefaultParser;

core/src/main/java/org/openapitools/openapidiff/core/compare/ApiResponseDiff.java

@@ -1,18 +1,17 @@
 package org.openapitools.openapidiff.core.compare;
 
-import static org.openapitools.openapidiff.core.utils.ChangedUtils.isChanged;
-
 import io.swagger.v3.oas.models.responses.ApiResponse;
 import io.swagger.v3.oas.models.responses.ApiResponses;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.Optional;
+import javax.annotation.Nullable;
+import org.openapitools.openapidiff.core.model.Changed;
 import org.openapitools.openapidiff.core.model.ChangedApiResponse;
 import org.openapitools.openapidiff.core.model.ChangedResponse;
 import org.openapitools.openapidiff.core.model.DiffContext;
-
-import javax.annotation.Nullable;
+import org.openapitools.openapidiff.core.model.deferred.DeferredBuilder;
+import org.openapitools.openapidiff.core.model.deferred.DeferredChanged;
 
 /** Created by adarsh.sharma on 04/01/18. */
 public class ApiResponseDiff {
@@ -22,25 +21,38 @@ public ApiResponseDiff(OpenApiDiff openApiDiff) {
     this.openApiDiff = openApiDiff;
   }
 
-  public Optional<ChangedApiResponse> diff(@Nullable  ApiResponses left, @Nullable ApiResponses right, DiffContext context) {
+  public DeferredChanged<ChangedApiResponse> diff(
+      @Nullable ApiResponses left, @Nullable ApiResponses right, DiffContext context) {
     MapKeyDiff<String, ApiResponse> responseMapKeyDiff = MapKeyDiff.diff(left, right);
     List<String> sharedResponseCodes = responseMapKeyDiff.getSharedKey();
     Map<String, ChangedResponse> resps = new LinkedHashMap<>();
+    DeferredBuilder<Changed> builder = new DeferredBuilder<>();
+
     for (String responseCode : sharedResponseCodes) {
-      openApiDiff
-          .getResponseDiff()
-          .diff(left != null ? left.get(responseCode) : null, right != null ? right.get(responseCode) : null, context)
+      builder
+          .with(
+              openApiDiff
+                  .getResponseDiff()
+                  .diff(
+                      left != null ? left.get(responseCode) : null,
+                      right != null ? right.get(responseCode) : null,
+                      context))
           .ifPresent(changedResponse -> resps.put(responseCode, changedResponse));
     }
     ChangedApiResponse changedApiResponse =
         new ChangedApiResponse(left, right, context)
             .setIncreased(responseMapKeyDiff.getIncreased())
             .setMissing(responseMapKeyDiff.getMissing())
             .setChanged(resps);
-    openApiDiff
-        .getExtensionsDiff()
-        .diff(left != null ? left.getExtensions() : null, right != null ? right.getExtensions() : null, context)
+    builder
+        .with(
+            openApiDiff
+                .getExtensionsDiff()
+                .diff(
+                    left != null ? left.getExtensions() : null,
+                    right != null ? right.getExtensions() : null,
+                    context))
         .ifPresent(changedApiResponse::setExtensions);
-    return isChanged(changedApiResponse);
+    return builder.buildIsChanged(changedApiResponse);
   }
 }

core/src/main/java/org/openapitools/openapidiff/core/compare/ContentDiff.java

@@ -6,9 +6,12 @@
 import io.swagger.v3.oas.models.media.Content;
 import io.swagger.v3.oas.models.media.MediaType;
 import java.util.*;
+import org.openapitools.openapidiff.core.model.Changed;
 import org.openapitools.openapidiff.core.model.ChangedContent;
 import org.openapitools.openapidiff.core.model.ChangedMediaType;
 import org.openapitools.openapidiff.core.model.DiffContext;
+import org.openapitools.openapidiff.core.model.deferred.DeferredBuilder;
+import org.openapitools.openapidiff.core.model.deferred.DeferredChanged;
 
 public class ContentDiff {
 
@@ -18,32 +21,47 @@ public ContentDiff(OpenApiDiff openApiDiff) {
     this.openApiDiff = openApiDiff;
   }
 
-  public Optional<ChangedContent> diff(Content left, Content right, DiffContext context) {
+  public DeferredChanged<ChangedContent> diff(Content left, Content right, DiffContext context) {
+    DeferredBuilder<Changed> builder = new DeferredBuilder<Changed>();
 
     MapKeyDiff<String, MediaType> mediaTypeDiff = MapKeyDiff.diff(left, right);
     List<String> sharedMediaTypes = mediaTypeDiff.getSharedKey();
     Map<String, ChangedMediaType> changedMediaTypes = new LinkedHashMap<>();
-    for (String mediaTypeKey : sharedMediaTypes) {
-      MediaType oldMediaType = left.get(mediaTypeKey);
-      MediaType newMediaType = right.get(mediaTypeKey);
-      ChangedMediaType changedMediaType =
-          new ChangedMediaType(oldMediaType.getSchema(), newMediaType.getSchema(), context);
-      openApiDiff
-          .getSchemaDiff()
-          .diff(
-              new HashSet<>(),
-              oldMediaType.getSchema(),
-              newMediaType.getSchema(),
-              context.copyWithRequired(true))
-          .ifPresent(changedMediaType::setSchema);
-      if (!isUnchanged(changedMediaType)) {
-        changedMediaTypes.put(mediaTypeKey, changedMediaType);
-      }
-    }
-    return isChanged(
-        new ChangedContent(left, right, context)
-            .setIncreased(mediaTypeDiff.getIncreased())
-            .setMissing(mediaTypeDiff.getMissing())
-            .setChanged(changedMediaTypes));
+
+    sharedMediaTypes.stream()
+        .forEach(
+            (mediaTypeKey) -> {
+              MediaType oldMediaType = left.get(mediaTypeKey);
+              MediaType newMediaType = right.get(mediaTypeKey);
+
+              ChangedMediaType changedMediaType =
+                  new ChangedMediaType(oldMediaType.getSchema(), newMediaType.getSchema(), context);
+
+              builder
+                  .with(
+                      openApiDiff
+                          .getSchemaDiff()
+                          .diff(
+                              oldMediaType.getSchema(),
+                              newMediaType.getSchema(),
+                              context.copyWithRequired(true)))
+                  .ifPresent(
+                      (value) -> {
+                        changedMediaType.setSchema(value);
+                        if (!isUnchanged(changedMediaType)) {
+                          changedMediaTypes.put(mediaTypeKey, changedMediaType);
+                        }
+                      });
+            });
+
+    return builder
+        .build()
+        .mapOptional(
+            (value) ->
+                isChanged(
+                    new ChangedContent(left, right, context)
+                        .setIncreased(mediaTypeDiff.getIncreased())
+                        .setMissing(mediaTypeDiff.getMissing())
+                        .setChanged(changedMediaTypes)));
   }
 }

core/src/main/java/org/openapitools/openapidiff/core/compare/HeaderDiff.java

@@ -1,14 +1,15 @@
 package org.openapitools.openapidiff.core.compare;
 
-import static org.openapitools.openapidiff.core.utils.ChangedUtils.isChanged;
-
 import io.swagger.v3.oas.models.Components;
 import io.swagger.v3.oas.models.headers.Header;
 import java.util.HashSet;
 import java.util.Objects;
 import java.util.Optional;
+import org.openapitools.openapidiff.core.model.Changed;
 import org.openapitools.openapidiff.core.model.ChangedHeader;
 import org.openapitools.openapidiff.core.model.DiffContext;
+import org.openapitools.openapidiff.core.model.deferred.DeferredBuilder;
+import org.openapitools.openapidiff.core.model.deferred.DeferredChanged;
 import org.openapitools.openapidiff.core.utils.RefPointer;
 import org.openapitools.openapidiff.core.utils.RefType;
 
@@ -31,16 +32,17 @@ public HeaderDiff(OpenApiDiff openApiDiff) {
             : null;
   }
 
-  public Optional<ChangedHeader> diff(Header left, Header right, DiffContext context) {
+  public DeferredChanged<ChangedHeader> diff(Header left, Header right, DiffContext context) {
     return cachedDiff(new HashSet<>(), left, right, left.get$ref(), right.get$ref(), context);
   }
 
   @Override
-  protected Optional<ChangedHeader> computeDiff(
+  protected DeferredChanged<ChangedHeader> computeDiff(
       HashSet<String> refSet, Header left, Header right, DiffContext context) {
     left = refPointer.resolveRef(leftComponents, left, left.get$ref());
     right = refPointer.resolveRef(rightComponents, right, right.get$ref());
 
+    DeferredBuilder<Changed> builder = new DeferredBuilder<Changed>();
     ChangedHeader changedHeader =
         new ChangedHeader(left, right, context)
             .setRequired(getBooleanDiff(left.getRequired(), right.getRequired()))
@@ -49,23 +51,28 @@ protected Optional<ChangedHeader> computeDiff(
                     && Boolean.TRUE.equals(right.getDeprecated()))
             .setStyle(!Objects.equals(left.getStyle(), right.getStyle()))
             .setExplode(getBooleanDiff(left.getExplode(), right.getExplode()));
-    openApiDiff
-        .getMetadataDiff()
-        .diff(left.getDescription(), right.getDescription(), context)
+    builder
+        .with(
+            openApiDiff
+                .getMetadataDiff()
+                .diff(left.getDescription(), right.getDescription(), context))
         .ifPresent(changedHeader::setDescription);
-    openApiDiff
-        .getSchemaDiff()
-        .diff(new HashSet<>(), left.getSchema(), right.getSchema(), context.copyWithRequired(true))
+    builder
+        .with(
+            openApiDiff
+                .getSchemaDiff()
+                .diff(left.getSchema(), right.getSchema(), context.copyWithRequired(true)))
         .ifPresent(changedHeader::setSchema);
-    openApiDiff
-        .getContentDiff()
-        .diff(left.getContent(), right.getContent(), context)
+    builder
+        .with(openApiDiff.getContentDiff().diff(left.getContent(), right.getContent(), context))
         .ifPresent(changedHeader::setContent);
-    openApiDiff
-        .getExtensionsDiff()
-        .diff(left.getExtensions(), right.getExtensions(), context)
+    builder
+        .with(
+            openApiDiff
+                .getExtensionsDiff()
+                .diff(left.getExtensions(), right.getExtensions(), context))
         .ifPresent(changedHeader::setExtensions);
-    return isChanged(changedHeader);
+    return builder.buildIsChanged(changedHeader);
   }
 
   private boolean getBooleanDiff(Boolean left, Boolean right) {

core/src/main/java/org/openapitools/openapidiff/core/compare/HeadersDiff.java

@@ -6,10 +6,11 @@
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.Optional;
 import org.openapitools.openapidiff.core.model.ChangedHeader;
 import org.openapitools.openapidiff.core.model.ChangedHeaders;
 import org.openapitools.openapidiff.core.model.DiffContext;
+import org.openapitools.openapidiff.core.model.deferred.DeferredBuilder;
+import org.openapitools.openapidiff.core.model.deferred.DeferredChanged;
 
 /** Created by adarsh.sharma on 28/12/17. */
 public class HeadersDiff {
@@ -19,24 +20,28 @@ public HeadersDiff(OpenApiDiff openApiDiff) {
     this.openApiDiff = openApiDiff;
   }
 
-  public Optional<ChangedHeaders> diff(
+  public DeferredChanged<ChangedHeaders> diff(
       Map<String, Header> left, Map<String, Header> right, DiffContext context) {
     MapKeyDiff<String, Header> headerMapDiff = MapKeyDiff.diff(left, right);
     List<String> sharedHeaderKeys = headerMapDiff.getSharedKey();
 
     Map<String, ChangedHeader> changed = new LinkedHashMap<>();
+    DeferredBuilder<ChangedHeader> builder = new DeferredBuilder<>();
     for (String headerKey : sharedHeaderKeys) {
       Header oldHeader = left.get(headerKey);
       Header newHeader = right.get(headerKey);
-      openApiDiff
-          .getHeaderDiff()
-          .diff(oldHeader, newHeader, context)
+      builder
+          .with(openApiDiff.getHeaderDiff().diff(oldHeader, newHeader, context))
           .ifPresent(changedHeader -> changed.put(headerKey, changedHeader));
     }
-    return isChanged(
-        new ChangedHeaders(left, right, context)
-            .setIncreased(headerMapDiff.getIncreased())
-            .setMissing(headerMapDiff.getMissing())
-            .setChanged(changed));
+    return builder
+        .build()
+        .mapOptional(
+            (value) ->
+                isChanged(
+                    new ChangedHeaders(left, right, context)
+                        .setIncreased(headerMapDiff.getIncreased())
+                        .setMissing(headerMapDiff.getMissing())
+                        .setChanged(changed)));
   }
 }

core/src/main/java/org/openapitools/openapidiff/core/compare/OpenApiDiff.java

@@ -8,14 +8,10 @@
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
-import java.util.Optional;
 import java.util.stream.Collectors;
-import org.openapitools.openapidiff.core.model.ChangedExtensions;
-import org.openapitools.openapidiff.core.model.ChangedOpenApi;
-import org.openapitools.openapidiff.core.model.ChangedOperation;
-import org.openapitools.openapidiff.core.model.ChangedPath;
-import org.openapitools.openapidiff.core.model.ChangedPaths;
-import org.openapitools.openapidiff.core.model.Endpoint;
+import org.openapitools.openapidiff.core.model.*;
+import org.openapitools.openapidiff.core.model.deferred.DeferredChanged;
+import org.openapitools.openapidiff.core.model.deferred.DeferredSchemaCache;
 import org.openapitools.openapidiff.core.utils.EndpointUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -48,6 +44,7 @@ public class OpenApiDiff {
   private List<Endpoint> missingEndpoints;
   private List<ChangedOperation> changedOperations;
   private ChangedExtensions changedExtensions;
+  private DeferredSchemaCache deferredSchemaCache;
 
   /*
    * @param oldSpecOpenApi
@@ -86,17 +83,25 @@ private void initializeFields() {
     this.oAuthFlowDiff = new OAuthFlowDiff(this);
     this.extensionsDiff = new ExtensionsDiff(this);
     this.metadataDiff = new MetadataDiff(this);
+    this.deferredSchemaCache = new DeferredSchemaCache(this);
   }
 
   private ChangedOpenApi compare() {
     preProcess(oldSpecOpenApi);
     preProcess(newSpecOpenApi);
-    Optional<ChangedPaths> paths =
+
+    // 1st pass scans paths to collect all schemas
+    DeferredChanged<ChangedPaths> paths =
         this.pathsDiff.diff(
             valOrEmpty(oldSpecOpenApi.getPaths()), valOrEmpty(newSpecOpenApi.getPaths()));
+
+    // 2nd pass processes deferred schemas
+    deferredSchemaCache.process();
+
     this.newEndpoints = new ArrayList<>();
     this.missingEndpoints = new ArrayList<>();
     this.changedOperations = new ArrayList<>();
+
     paths.ifPresent(
         changedPaths -> {
           this.newEndpoints = EndpointUtils.convert2EndpointList(changedPaths.getIncreased());
@@ -117,6 +122,7 @@ private ChangedOpenApi compare() {
     getExtensionsDiff()
         .diff(oldSpecOpenApi.getExtensions(), newSpecOpenApi.getExtensions())
         .ifPresent(this::setChangedExtension);
+
     return getChangedOpenApi();
   }
 
@@ -162,7 +168,12 @@ private ChangedOpenApi getChangedOpenApi() {
         .setNewSpecOpenApi(newSpecOpenApi)
         .setOldSpecOpenApi(oldSpecOpenApi)
         .setChangedOperations(changedOperations)
-        .setChangedExtensions(changedExtensions);
+        .setChangedExtensions(changedExtensions)
+        .setChangedSchemas(deferredSchemaCache.getChangedSchemas());
+  }
+
+  public DeferredSchemaCache getDeferredSchemaCache() {
+    return deferredSchemaCache;
   }
 
   public PathsDiff getPathsDiff() {