Skip to content

Commit 36b9c20

Browse files
author
AWS
committed
AWS Glue Update: Feature1 - Glue crawler adds data lineage configuration option. Feature2 - AWS Glue Data Catalog adds APIs for PartitionIndex creation and deletion as part of Enhancement Partition Management feature.
1 parent 5c360da commit 36b9c20

File tree

2 files changed

+177
-2
lines changed

2 files changed

+177
-2
lines changed

‎.changes/next-release/feature-AWSGlue-4e90067.json

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
{
2+
"type": "feature",
3+
"category": "AWS Glue",
4+
"contributor": "",
5+
"description": "Feature1 - Glue crawler adds data lineage configuration option. Feature2 - AWS Glue Data Catalog adds APIs for PartitionIndex creation and deletion as part of Enhancement Partition Management feature."
6+
}

‎services/glue/src/main/resources/codegen-resources/service-2.json

Lines changed: 171 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -391,6 +391,25 @@
391391
],
392392
"documentation":"<p>Creates a new partition.</p>"
393393
},
394+
"CreatePartitionIndex":{
395+
"name":"CreatePartitionIndex",
396+
"http":{
397+
"method":"POST",
398+
"requestUri":"/"
399+
},
400+
"input":{"shape":"CreatePartitionIndexRequest"},
401+
"output":{"shape":"CreatePartitionIndexResponse"},
402+
"errors":[
403+
{"shape":"AlreadyExistsException"},
404+
{"shape":"InvalidInputException"},
405+
{"shape":"EntityNotFoundException"},
406+
{"shape":"ResourceNumberLimitExceededException"},
407+
{"shape":"InternalServiceException"},
408+
{"shape":"OperationTimeoutException"},
409+
{"shape":"GlueEncryptionException"}
410+
],
411+
"documentation":"<p>Creates a specified partition index in an existing table.</p>"
412+
},
394413
"CreateRegistry":{
395414
"name":"CreateRegistry",
396415
"http":{
@@ -691,6 +710,24 @@
691710
],
692711
"documentation":"<p>Deletes a specified partition.</p>"
693712
},
713+
"DeletePartitionIndex":{
714+
"name":"DeletePartitionIndex",
715+
"http":{
716+
"method":"POST",
717+
"requestUri":"/"
718+
},
719+
"input":{"shape":"DeletePartitionIndexRequest"},
720+
"output":{"shape":"DeletePartitionIndexResponse"},
721+
"errors":[
722+
{"shape":"InternalServiceException"},
723+
{"shape":"OperationTimeoutException"},
724+
{"shape":"InvalidInputException"},
725+
{"shape":"EntityNotFoundException"},
726+
{"shape":"ConflictException"},
727+
{"shape":"GlueEncryptionException"}
728+
],
729+
"documentation":"<p>Deletes a specified partition index from an existing table.</p>"
730+
},
694731
"DeleteRegistry":{
695732
"name":"DeleteRegistry",
696733
"http":{
@@ -2603,6 +2640,38 @@
26032640
"exception":true
26042641
},
26052642
"AttemptCount":{"type":"integer"},
2643+
"BackfillError":{
2644+
"type":"structure",
2645+
"members":{
2646+
"Code":{
2647+
"shape":"BackfillErrorCode",
2648+
"documentation":"<p>The error code for an error that occurred when registering partition indexes for an existing table.</p>"
2649+
},
2650+
"Partitions":{
2651+
"shape":"BackfillErroredPartitionsList",
2652+
"documentation":"<p>A list of a limited number of partitions in the response.</p>"
2653+
}
2654+
},
2655+
"documentation":"<p>A list of errors that can occur when registering partition indexes for an existing table.</p> <p>These errors give the details about why an index registration failed and provide a limited number of partitions in the response, so that you can fix the partitions at fault and try registering the index again. The most common set of errors that can occur are categorized as follows:</p> <ul> <li> <p>EncryptedPartitionError: The partitions are encrypted.</p> </li> <li> <p>InvalidPartitionTypeDataError: The partition value doesn't match the data type for that partition column.</p> </li> <li> <p>MissingPartitionValueError: The partitions are encrypted.</p> </li> <li> <p>UnsupportedPartitionCharacterError: Characters inside the partition value are not supported. For example: U+0000 , U+0001, U+0002.</p> </li> <li> <p>InternalError: Any error which does not belong to other error codes.</p> </li> </ul>"
2656+
},
2657+
"BackfillErrorCode":{
2658+
"type":"string",
2659+
"enum":[
2660+
"ENCRYPTED_PARTITION_ERROR",
2661+
"INTERNAL_ERROR",
2662+
"INVALID_PARTITION_TYPE_DATA_ERROR",
2663+
"MISSING_PARTITION_VALUE_ERROR",
2664+
"UNSUPPORTED_PARTITION_CHARACTER_ERROR"
2665+
]
2666+
},
2667+
"BackfillErroredPartitionsList":{
2668+
"type":"list",
2669+
"member":{"shape":"PartitionValueList"}
2670+
},
2671+
"BackfillErrors":{
2672+
"type":"list",
2673+
"member":{"shape":"BackfillError"}
2674+
},
26062675
"BatchCreatePartitionRequest":{
26072676
"type":"structure",
26082677
"required":[
@@ -3955,6 +4024,10 @@
39554024
"shape":"SchemaChangePolicy",
39564025
"documentation":"<p>The policy that specifies update and delete behaviors for the crawler.</p>"
39574026
},
4027+
"LineageConfiguration":{
4028+
"shape":"LineageConfiguration",
4029+
"documentation":"<p>A configuration that specifies whether data lineage is enabled for the crawler.</p>"
4030+
},
39584031
"State":{
39594032
"shape":"CrawlerState",
39604033
"documentation":"<p>Indicates whether the crawler is running, or whether a run is pending.</p>"
@@ -3999,6 +4072,13 @@
39994072
"documentation":"<p>Specifies a crawler program that examines a data source and uses classifiers to try to determine its schema. If successful, the crawler records metadata concerning the data source in the AWS Glue Data Catalog.</p>"
40004073
},
40014074
"CrawlerConfiguration":{"type":"string"},
4075+
"CrawlerLineageSettings":{
4076+
"type":"string",
4077+
"enum":[
4078+
"ENABLE",
4079+
"DISABLE"
4080+
]
4081+
},
40024082
"CrawlerList":{
40034083
"type":"list",
40044084
"member":{"shape":"Crawler"}
@@ -4226,6 +4306,10 @@
42264306
"shape":"RecrawlPolicy",
42274307
"documentation":"<p>A policy that specifies whether to crawl the entire dataset again, or to crawl only folders that were added since the last crawler run.</p>"
42284308
},
4309+
"LineageConfiguration":{
4310+
"shape":"LineageConfiguration",
4311+
"documentation":"<p>Specifies data lineage configuration settings for the crawler.</p>"
4312+
},
42294313
"Configuration":{
42304314
"shape":"CrawlerConfiguration",
42314315
"documentation":"<p>Crawler configuration information. This versioned JSON string allows users to specify aspects of a crawler's behavior. For more information, see <a href=\"https://docs.aws.amazon.com/glue/latest/dg/crawler-configuration.html\">Configuring a Crawler</a>.</p>"
@@ -4663,6 +4747,37 @@
46634747
}
46644748
}
46654749
},
4750+
"CreatePartitionIndexRequest":{
4751+
"type":"structure",
4752+
"required":[
4753+
"DatabaseName",
4754+
"TableName",
4755+
"PartitionIndex"
4756+
],
4757+
"members":{
4758+
"CatalogId":{
4759+
"shape":"CatalogIdString",
4760+
"documentation":"<p>The catalog ID where the table resides.</p>"
4761+
},
4762+
"DatabaseName":{
4763+
"shape":"NameString",
4764+
"documentation":"<p>Specifies the name of a database in which you want to create a partition index.</p>"
4765+
},
4766+
"TableName":{
4767+
"shape":"NameString",
4768+
"documentation":"<p>Specifies the name of a table in which you want to create a partition index.</p>"
4769+
},
4770+
"PartitionIndex":{
4771+
"shape":"PartitionIndex",
4772+
"documentation":"<p>Specifies a <code>PartitionIndex</code> structure to create a partition index in an existing table.</p>"
4773+
}
4774+
}
4775+
},
4776+
"CreatePartitionIndexResponse":{
4777+
"type":"structure",
4778+
"members":{
4779+
}
4780+
},
46664781
"CreatePartitionRequest":{
46674782
"type":"structure",
46684783
"required":[
@@ -5540,6 +5655,37 @@
55405655
}
55415656
}
55425657
},
5658+
"DeletePartitionIndexRequest":{
5659+
"type":"structure",
5660+
"required":[
5661+
"DatabaseName",
5662+
"TableName",
5663+
"IndexName"
5664+
],
5665+
"members":{
5666+
"CatalogId":{
5667+
"shape":"CatalogIdString",
5668+
"documentation":"<p>The catalog ID where the table resides.</p>"
5669+
},
5670+
"DatabaseName":{
5671+
"shape":"NameString",
5672+
"documentation":"<p>Specifies the name of a database from which you want to delete a partition index.</p>"
5673+
},
5674+
"TableName":{
5675+
"shape":"NameString",
5676+
"documentation":"<p>Specifies the name of a table from which you want to delete a partition index.</p>"
5677+
},
5678+
"IndexName":{
5679+
"shape":"NameString",
5680+
"documentation":"<p>The name of the partition index to be deleted.</p>"
5681+
}
5682+
}
5683+
},
5684+
"DeletePartitionIndexResponse":{
5685+
"type":"structure",
5686+
"members":{
5687+
}
5688+
},
55435689
"DeletePartitionRequest":{
55445690
"type":"structure",
55455691
"required":[
@@ -8712,6 +8858,16 @@
87128858
]
87138859
},
87148860
"LatestSchemaVersionBoolean":{"type":"boolean"},
8861+
"LineageConfiguration":{
8862+
"type":"structure",
8863+
"members":{
8864+
"CrawlerLineageSettings":{
8865+
"shape":"CrawlerLineageSettings",
8866+
"documentation":"<p>Specifies whether data lineage is enabled for the crawler. Valid values are:</p> <ul> <li> <p>ENABLE: enables data lineage for the crawler</p> </li> <li> <p>DISABLE: disables data lineage for the crawler</p> </li> </ul>"
8867+
}
8868+
},
8869+
"documentation":"<p>Specifies data lineage configuration settings for the crawler.</p>"
8870+
},
87158871
"ListCrawlersRequest":{
87168872
"type":"structure",
87178873
"members":{
@@ -9570,7 +9726,11 @@
95709726
},
95719727
"IndexStatus":{
95729728
"shape":"PartitionIndexStatus",
9573-
"documentation":"<p>The status of the partition index. </p>"
9729+
"documentation":"<p>The status of the partition index. </p> <p>The possible statuses are:</p> <ul> <li> <p>CREATING: The index is being created. When an index is in a CREATING state, the index or its table cannot be deleted.</p> </li> <li> <p>ACTIVE: The index creation succeeds.</p> </li> <li> <p>FAILED: The index creation fails. </p> </li> <li> <p>DELETING: The index is deleted from the list of indexes.</p> </li> </ul>"
9730+
},
9731+
"BackfillErrors":{
9732+
"shape":"BackfillErrors",
9733+
"documentation":"<p>A list of errors that can occur when registering partition indexes for an existing table.</p>"
95749734
}
95759735
},
95769736
"documentation":"<p>A descriptor for a partition index in a table.</p>"
@@ -9586,7 +9746,12 @@
95869746
},
95879747
"PartitionIndexStatus":{
95889748
"type":"string",
9589-
"enum":["ACTIVE"]
9749+
"enum":[
9750+
"CREATING",
9751+
"ACTIVE",
9752+
"DELETING",
9753+
"FAILED"
9754+
]
95909755
},
95919756
"PartitionInput":{
95929757
"type":"structure",
@@ -11990,6 +12155,10 @@
1199012155
"shape":"RecrawlPolicy",
1199112156
"documentation":"<p>A policy that specifies whether to crawl the entire dataset again, or to crawl only folders that were added since the last crawler run.</p>"
1199212157
},
12158+
"LineageConfiguration":{
12159+
"shape":"LineageConfiguration",
12160+
"documentation":"<p>Specifies data lineage configuration settings for the crawler.</p>"
12161+
},
1199312162
"Configuration":{
1199412163
"shape":"CrawlerConfiguration",
1199512164
"documentation":"<p>Crawler configuration information. This versioned JSON string allows users to specify aspects of a crawler's behavior. For more information, see <a href=\"https://docs.aws.amazon.com/glue/latest/dg/crawler-configuration.html\">Configuring a Crawler</a>.</p>"

0 commit comments

Comments
 (0)