From 7207f8e545f21d1b00b02e52487b91ac41b0560c Mon Sep 17 00:00:00 2001 From: Microsoft Graph DevX Tooling Date: Thu, 13 Feb 2025 17:54:00 +0300 Subject: [PATCH 01/11] Added regex that ensures null valued properties are not omitted --- src/readme.graph.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/readme.graph.md b/src/readme.graph.md index a2c0dd987e..8f335256f2 100644 --- a/src/readme.graph.md +++ b/src/readme.graph.md @@ -335,6 +335,11 @@ directive: let dateTimeToJsonRegex = /(\.Json\.JsonString\()(.*)\?(\.ToString\(@"yyyy'-'MM'-'dd'T'HH':'mm':'ss\.fffffffK")/gm $ = $.replace(dateTimeToJsonRegex, '$1System.DateTime.SpecifyKind($2.Value.ToUniversalTime(), System.DateTimeKind.Utc)$3'); + // Enables null valued properties + $ = $.replace(/AddIf\(\s*null\s*!=\s*(this\._\w+)\s*\?\s*\(\s*Microsoft\.Graph\.PowerShell\.Runtime\.Json\.JsonNode\)\s*(.*)\s*:\s*null\s*,\s*"(.*?)"\s*,\s*container\.Add\s*\)/gm, 'container.Add("$3", $1 != null ? (Microsoft.Graph.PowerShell.Runtime.Json.JsonNode) $2 :"defaultnull")') + + $ = $.replace(/AddIf\(\s*null\s*!=\s*\(\(\(\(object\)\s*(this\._\w+)\)\)?.ToString\(\)\)\s*\?\s*\(\s*Microsoft\.Graph\.PowerShell\.Runtime\.Json\.JsonNode\)\s*new\s*Microsoft\.Graph\.PowerShell\.Runtime\.Json\.JsonString\((this\._\w+).ToString\(\)\)\s*:\s*null\s*,\s*"(.*?)"\s*,\s*container\.Add\s*\)/gm, 'container.Add("$3", $1 != null ? (Microsoft.Graph.PowerShell.Runtime.Json.JsonNode) new Microsoft.Graph.PowerShell.Runtime.Json.JsonString($2.ToString()) :"defaultnull)'); + return $; } # Modify generated .dictionary.cs model classes. From f4df01fbeb65533ef0627abb483267008d078b39 Mon Sep 17 00:00:00 2001 From: Microsoft Graph DevX Tooling Date: Thu, 13 Feb 2025 18:33:36 +0300 Subject: [PATCH 02/11] Added JsonExtensions class --- src/readme.graph.md | 4 +-- tools/Custom/JsonExtensions.cs | 63 ++++++++++++++++++++++++++++++++++ 2 files changed, 65 insertions(+), 2 deletions(-) create mode 100644 tools/Custom/JsonExtensions.cs diff --git a/src/readme.graph.md b/src/readme.graph.md index 8f335256f2..f3d971af85 100644 --- a/src/readme.graph.md +++ b/src/readme.graph.md @@ -337,8 +337,8 @@ directive: // Enables null valued properties $ = $.replace(/AddIf\(\s*null\s*!=\s*(this\._\w+)\s*\?\s*\(\s*Microsoft\.Graph\.PowerShell\.Runtime\.Json\.JsonNode\)\s*(.*)\s*:\s*null\s*,\s*"(.*?)"\s*,\s*container\.Add\s*\)/gm, 'container.Add("$3", $1 != null ? (Microsoft.Graph.PowerShell.Runtime.Json.JsonNode) $2 :"defaultnull")') - - $ = $.replace(/AddIf\(\s*null\s*!=\s*\(\(\(\(object\)\s*(this\._\w+)\)\)?.ToString\(\)\)\s*\?\s*\(\s*Microsoft\.Graph\.PowerShell\.Runtime\.Json\.JsonNode\)\s*new\s*Microsoft\.Graph\.PowerShell\.Runtime\.Json\.JsonString\((this\._\w+).ToString\(\)\)\s*:\s*null\s*,\s*"(.*?)"\s*,\s*container\.Add\s*\)/gm, 'container.Add("$3", $1 != null ? (Microsoft.Graph.PowerShell.Runtime.Json.JsonNode) new Microsoft.Graph.PowerShell.Runtime.Json.JsonString($2.ToString()) :"defaultnull)'); + + $ = $.replace(/AddIf\(\s*null\s*!=\s*\(\(\(\(object\)\s*(this\._\w+)\)\)?.ToString\(\)\)\s*\?\s*\(\s*Microsoft\.Graph\.PowerShell\.Runtime\.Json\.JsonNode\)\s*new\s*Microsoft\.Graph\.PowerShell\.Runtime\.Json\.JsonString\((this\._\w+).ToString\(\)\)\s*:\s*null\s*,\s*"(.*?)"\s*,\s*container\.Add\s*\)/gm, 'container.Add("$3", $1 != null ? (Microsoft.Graph.PowerShell.Runtime.Json.JsonNode) new Microsoft.Graph.PowerShell.Runtime.Json.JsonString($2.ToString()) :"defaultnull")'); return $; } diff --git a/tools/Custom/JsonExtensions.cs b/tools/Custom/JsonExtensions.cs new file mode 100644 index 0000000000..621f4eedb6 --- /dev/null +++ b/tools/Custom/JsonExtensions.cs @@ -0,0 +1,63 @@ +namespace Microsoft.Graph.PowerShell.JsonUtilities +{ + using Newtonsoft.Json.Linq; + using System.Linq; + using static Microsoft.Graph.PowerShell.Runtime.Extensions; + + public static class JsonExtensions + { + /// + /// Removes JSON properties that have a value of "defaultnull" and converts properties with values of "null" or empty strings ("") to actual JSON null values. + /// + /// The JObject to process and clean. + /// + /// A JSON string representation of the cleaned JObject with "defaultnull" properties removed and "null" or empty string values converted to JSON null. + /// + /// + /// JObject json = JObject.Parse(@"{""name"": ""John"", ""email"": ""defaultnull"", ""address"": ""null""}"); + /// string cleanedJson = json.RemoveDefaultNullProperties(); + /// Console.WriteLine(cleanedJson); + /// // Output: { "name": "John", "address": null } + /// + public static string RemoveDefaultNullProperties(this JObject jsonObject) + { + try + { + foreach (var property in jsonObject.Properties().ToList()) + { + if (property.Value.Type == JTokenType.Object) + { + RemoveDefaultNullProperties((JObject)property.Value); + } + else if (property.Value.Type == JTokenType.Array) + { + foreach (var item in property.Value) + { + if (item.Type == JTokenType.Object) + { + RemoveDefaultNullProperties((JObject)item); + } + } + } + else if (property.Value.Type == JTokenType.String && property.Value.ToString() == "defaultnull") + { + property.Remove(); + } + else if (property.Value.Type == JTokenType.String && (property.Value.ToString() == "null" || property.Value.ToString() == "")) + { + property.Value = JValue.CreateNull(); + } + } + } + catch (System.Exception) + { + return jsonObject.ToString(); // Return the original string if parsing fails + } + return jsonObject.ToString(); + } + public static string ReplaceAndRemoveSlashes(this string body) + { + return body.Replace("/", "").Replace("\\", "").Replace("rn", "").Replace("\"{", "{").Replace("}\"", "}"); + } + } +} From 21d997646d81c191ee86475180d069b85bfd9388 Mon Sep 17 00:00:00 2001 From: Microsoft Graph DevX Tooling Date: Thu, 13 Feb 2025 19:49:58 +0300 Subject: [PATCH 03/11] Updated test to cinclude file from custom folder --- tools/Custom/JsonExtensions.cs | 1 - .../JsonUtilitiesTest/JsonExtensionsTests.cs | 128 ++++++++++++++++++ .../JsonUtilitiesTest.csproj | 27 ++++ 3 files changed, 155 insertions(+), 1 deletion(-) create mode 100644 tools/Tests/JsonUtilitiesTest/JsonExtensionsTests.cs create mode 100644 tools/Tests/JsonUtilitiesTest/JsonUtilitiesTest.csproj diff --git a/tools/Custom/JsonExtensions.cs b/tools/Custom/JsonExtensions.cs index 621f4eedb6..3af71b286f 100644 --- a/tools/Custom/JsonExtensions.cs +++ b/tools/Custom/JsonExtensions.cs @@ -2,7 +2,6 @@ namespace Microsoft.Graph.PowerShell.JsonUtilities { using Newtonsoft.Json.Linq; using System.Linq; - using static Microsoft.Graph.PowerShell.Runtime.Extensions; public static class JsonExtensions { diff --git a/tools/Tests/JsonUtilitiesTest/JsonExtensionsTests.cs b/tools/Tests/JsonUtilitiesTest/JsonExtensionsTests.cs new file mode 100644 index 0000000000..6fd5dba519 --- /dev/null +++ b/tools/Tests/JsonUtilitiesTest/JsonExtensionsTests.cs @@ -0,0 +1,128 @@ +namespace JsonUtilitiesTest; +using System; +using Newtonsoft.Json.Linq; +using Xunit; +using Microsoft.Graph.PowerShell.JsonUtilities; + +public class JsonExtensionsTests +{ + [Fact] + public void RemoveDefaultNullProperties_ShouldRemoveDefaultNullValues() + { + // Arrange + JObject json = JObject.Parse(@"{ + ""displayname"": ""Tim"", + ""position"": ""defaultnull"", + ""salary"": 2000000, + ""team"": ""defaultnull"" + }"); + + // Act + string cleanedJson = json.RemoveDefaultNullProperties(); + JObject result = JObject.Parse(cleanedJson); + + // Assert + Assert.False(result.ContainsKey("position")); + Assert.False(result.ContainsKey("team")); + Assert.Equal("Tim", result["displayname"]?.ToString()); + Assert.Equal(2000000, result["salary"]?.ToObject()); + } + + [Fact] + public void RemoveDefaultNullProperties_ShouldConvertStringNullToJsonNull() + { + // Arrange + JObject json = JObject.Parse(@"{ + ""displayname"": ""Tim"", + ""position"": ""null"", + ""salary"": 2000000, + ""team"": """" + }"); + + // Act + string cleanedJson = json.RemoveDefaultNullProperties(); + JObject result = JObject.Parse(cleanedJson); + + // Assert + Assert.Null(result["position"]?.Value()); + Assert.Null(result["team"]?.Value()); + Assert.Equal("Tim", result["displayname"]?.ToString()); + Assert.Equal(2000000, result["salary"]?.ToObject()); + } + + [Fact] + public void RemoveDefaultNullProperties_ShouldHandleNestedObjects() + { + // Arrange + JObject json = JObject.Parse(@"{ + ""displayname"": ""Tim"", + ""metadata"": { + ""phone"": ""defaultnull"", + ""location"": ""Nairobi"" + } + }"); + + // Act + string cleanedJson = json.RemoveDefaultNullProperties(); + JObject result = JObject.Parse(cleanedJson); + + // Assert + Assert.False(result["metadata"]?.ToObject()?.ContainsKey("phone")); + Assert.Equal("Nairobi", result["metadata"]?["location"]?.ToString()); + } + + [Fact] + public void RemoveDefaultNullProperties_ShouldHandleEmptyJsonObject() + { + // Arrange + JObject json = JObject.Parse(@"{}"); + + // Act + string cleanedJson = json.RemoveDefaultNullProperties(); + JObject result = JObject.Parse(cleanedJson); + + // Assert + Assert.Empty(result); + } + + [Fact] + public void RemoveDefaultNullProperties_ShouldHandleJsonArrays() + { + // Arrange + JObject json = JObject.Parse(@"{ + ""users"": [ + { ""displayname"": ""Tim"", ""email"": ""defaultnull"" }, + { ""displayname"": ""Mayabi"", ""email"": ""mayabi@example.com"" } + ] + }"); + + // Act + string cleanedJson = json.RemoveDefaultNullProperties(); + JObject result = JObject.Parse(cleanedJson); + + // Assert + Assert.Equal("Tim", result["users"]?[0]?["displayname"]?.ToString()); + Assert.Equal("mayabi@example.com", result["users"]?[1]?["email"]?.ToString()); + } + + [Fact] + public void RemoveDefaultNullProperties_ShouldNotAlterValidData() + { + // Arrange + JObject json = JObject.Parse(@"{ + ""displayname"": ""Tim"", + ""email"": ""mayabi@example.com"", + ""salary"": 2000000 + }"); + + // Act + string cleanedJson = json.RemoveDefaultNullProperties(); + JObject result = JObject.Parse(cleanedJson); + + // Assert + Assert.Equal("Tim", result["displayname"]?.ToString()); + Assert.Equal("mayabi@example.com", result["email"]?.ToString()); + Assert.Equal(2000000, result["salary"]?.ToObject()); + } +} + diff --git a/tools/Tests/JsonUtilitiesTest/JsonUtilitiesTest.csproj b/tools/Tests/JsonUtilitiesTest/JsonUtilitiesTest.csproj new file mode 100644 index 0000000000..2db61912cf --- /dev/null +++ b/tools/Tests/JsonUtilitiesTest/JsonUtilitiesTest.csproj @@ -0,0 +1,27 @@ + + + + net8.0 + enable + enable + false + + + + + + + + + + + + + + + + ../../Custom/JsonExtensions.cs + + + + From d2b97421655d1208d930ea9deabccf40be0f9bee Mon Sep 17 00:00:00 2001 From: Microsoft Graph DevX Tooling Date: Thu, 13 Feb 2025 23:37:02 +0300 Subject: [PATCH 04/11] Added tests to pipeline --- .../generation-templates/authentication-module.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.azure-pipelines/generation-templates/authentication-module.yml b/.azure-pipelines/generation-templates/authentication-module.yml index 86b7171a52..e94da3931e 100644 --- a/.azure-pipelines/generation-templates/authentication-module.yml +++ b/.azure-pipelines/generation-templates/authentication-module.yml @@ -30,6 +30,15 @@ steps: script: | . $(System.DefaultWorkingDirectory)/tools/GenerateAuthenticationModule.ps1 -Test + - ${{ if eq(parameters.Test, true) }}: + - task: PowerShell@2 + displayName: Test Json Utilities + inputs: + pwsh: true + targetType: inline + script: dotnet test + workingDirectory: "$(System.DefaultWorkingDirectory)/tools/Tests/JsonUtilitiesTests" + - ${{ if eq(parameters.Sign, true) }}: - template: ../common-templates/esrp/strongname.yml parameters: From 2e95743e45614c387eabd2b88a1e30c962187976 Mon Sep 17 00:00:00 2001 From: Tim Date: Fri, 14 Feb 2025 07:55:53 +0000 Subject: [PATCH 05/11] Removed conversion of empty strings to null --- tools/Custom/JsonExtensions.cs | 6 +++--- tools/Tests/JsonUtilitiesTest/JsonExtensionsTests.cs | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/Custom/JsonExtensions.cs b/tools/Custom/JsonExtensions.cs index 3af71b286f..0c0a69d888 100644 --- a/tools/Custom/JsonExtensions.cs +++ b/tools/Custom/JsonExtensions.cs @@ -6,11 +6,11 @@ namespace Microsoft.Graph.PowerShell.JsonUtilities public static class JsonExtensions { /// - /// Removes JSON properties that have a value of "defaultnull" and converts properties with values of "null" or empty strings ("") to actual JSON null values. + /// Removes JSON properties that have a value of "defaultnull" and converts properties with values of "null" to actual JSON null values. /// /// The JObject to process and clean. /// - /// A JSON string representation of the cleaned JObject with "defaultnull" properties removed and "null" or empty string values converted to JSON null. + /// A JSON string representation of the cleaned JObject with "defaultnull" properties removed and "null" values converted to JSON null. /// /// /// JObject json = JObject.Parse(@"{""name"": ""John"", ""email"": ""defaultnull"", ""address"": ""null""}"); @@ -42,7 +42,7 @@ public static string RemoveDefaultNullProperties(this JObject jsonObject) { property.Remove(); } - else if (property.Value.Type == JTokenType.String && (property.Value.ToString() == "null" || property.Value.ToString() == "")) + else if (property.Value.Type == JTokenType.String && (property.Value.ToString() == "null")) { property.Value = JValue.CreateNull(); } diff --git a/tools/Tests/JsonUtilitiesTest/JsonExtensionsTests.cs b/tools/Tests/JsonUtilitiesTest/JsonExtensionsTests.cs index 6fd5dba519..ed45257e94 100644 --- a/tools/Tests/JsonUtilitiesTest/JsonExtensionsTests.cs +++ b/tools/Tests/JsonUtilitiesTest/JsonExtensionsTests.cs @@ -45,7 +45,7 @@ public void RemoveDefaultNullProperties_ShouldConvertStringNullToJsonNull() // Assert Assert.Null(result["position"]?.Value()); - Assert.Null(result["team"]?.Value()); + Assert.Equal("",result["team"]?.ToString()); Assert.Equal("Tim", result["displayname"]?.ToString()); Assert.Equal(2000000, result["salary"]?.ToObject()); } From cbf6357ec8334520dee22a6a266d7dacf44a0ccb Mon Sep 17 00:00:00 2001 From: Microsoft Graph DevX Tooling Date: Fri, 14 Feb 2025 11:32:33 +0300 Subject: [PATCH 06/11] Updated submodule --- autorest.powershell | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/autorest.powershell b/autorest.powershell index 1865c6e579..bcc1ebb035 160000 --- a/autorest.powershell +++ b/autorest.powershell @@ -1 +1 @@ -Subproject commit 1865c6e579269c45a45ad1c6cf6d856b6d27643f +Subproject commit bcc1ebb035cccd0f0e2fef839182cd8a7bcc4985 From 5c427f772f2ddfbff9017ecdfbdf2a79fa9ae484 Mon Sep 17 00:00:00 2001 From: Microsoft Graph DevX Tooling Date: Fri, 14 Feb 2025 11:36:55 +0300 Subject: [PATCH 07/11] Fixed typo on path --- .azure-pipelines/generation-templates/authentication-module.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure-pipelines/generation-templates/authentication-module.yml b/.azure-pipelines/generation-templates/authentication-module.yml index e94da3931e..4dc2d7148d 100644 --- a/.azure-pipelines/generation-templates/authentication-module.yml +++ b/.azure-pipelines/generation-templates/authentication-module.yml @@ -37,7 +37,7 @@ steps: pwsh: true targetType: inline script: dotnet test - workingDirectory: "$(System.DefaultWorkingDirectory)/tools/Tests/JsonUtilitiesTests" + workingDirectory: "$(System.DefaultWorkingDirectory)/tools/Tests/JsonUtilitiesTest" - ${{ if eq(parameters.Sign, true) }}: - template: ../common-templates/esrp/strongname.yml From 2a7a620e56b7320a52dfcf13e95b63663d3492cd Mon Sep 17 00:00:00 2001 From: Microsoft Graph DevX Tooling Date: Fri, 14 Feb 2025 12:31:49 +0300 Subject: [PATCH 08/11] Updated submodule --- autorest.powershell | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/autorest.powershell b/autorest.powershell index bcc1ebb035..37df70c052 160000 --- a/autorest.powershell +++ b/autorest.powershell @@ -1 +1 @@ -Subproject commit bcc1ebb035cccd0f0e2fef839182cd8a7bcc4985 +Subproject commit 37df70c05228568a589ea8cbf6aafcc515dce003 From 31df688cabada42f04f363708341d419beef0d23 Mon Sep 17 00:00:00 2001 From: Microsoft Graph DevX Tooling Date: Fri, 14 Feb 2025 17:07:47 +0300 Subject: [PATCH 09/11] Updates submodule --- autorest.powershell | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/autorest.powershell b/autorest.powershell index 37df70c052..5b8bbf1c78 160000 --- a/autorest.powershell +++ b/autorest.powershell @@ -1 +1 @@ -Subproject commit 37df70c05228568a589ea8cbf6aafcc515dce003 +Subproject commit 5b8bbf1c784579cb441c99fd3936e111fd69d0e1 From 1bf42db81e8def1e287182ebe2d2927c367f11f3 Mon Sep 17 00:00:00 2001 From: Microsoft Graph DevX Tooling Date: Fri, 14 Feb 2025 17:54:25 +0300 Subject: [PATCH 10/11] Update submodule --- autorest.powershell | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/autorest.powershell b/autorest.powershell index 5b8bbf1c78..03c616edec 160000 --- a/autorest.powershell +++ b/autorest.powershell @@ -1 +1 @@ -Subproject commit 5b8bbf1c784579cb441c99fd3936e111fd69d0e1 +Subproject commit 03c616edecf4b7168b3b6a921d6a2515aa31099a From 9da380451c1997b6b1eb1a2db4eaeb430cd80a66 Mon Sep 17 00:00:00 2001 From: Microsoft Graph DevX Tooling Date: Fri, 14 Feb 2025 18:31:05 +0300 Subject: [PATCH 11/11] Updated submodule --- autorest.powershell | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/autorest.powershell b/autorest.powershell index 03c616edec..e2dd039108 160000 --- a/autorest.powershell +++ b/autorest.powershell @@ -1 +1 @@ -Subproject commit 03c616edecf4b7168b3b6a921d6a2515aa31099a +Subproject commit e2dd0391085d76318d6f19001dce7938eb8281e1