Skip to content

Commit 6ebfdf3

Browse files
Mpdreamzrusscam
Mpdreamz
authored and
russcam
committed
Update code generator so that we can check in XPACK specs
1 parent 4038910 commit 6ebfdf3

31 files changed

+2691
-4
lines changed

‎src/CodeGeneration/CodeGeneration.LowLevelClient/CodeConfiguration.cs

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
using System.Collections.Generic;
2+
using System.IO;
3+
using System.Linq;
4+
using System.Reflection;
5+
using System.Text.RegularExpressions;
6+
7+
namespace CodeGeneration.LowLevelClient
8+
{
9+
public static class CodeConfiguration
10+
{
11+
public static readonly Assembly Assembly = typeof(ApiGenerator).Assembly;
12+
13+
private static string _root = null;
14+
15+
private static string Root
16+
{
17+
get
18+
{
19+
if (CodeConfiguration._root != null) return CodeConfiguration._root;
20+
var directoryInfo = new DirectoryInfo(Directory.GetCurrentDirectory());
21+
22+
var runningAsDnx =
23+
directoryInfo.Name == "CodeGeneration.LowLevelClient" &&
24+
directoryInfo.Parent != null &&
25+
directoryInfo.Parent.Name == "CodeGeneration";
26+
27+
CodeConfiguration._root = runningAsDnx ? "" : @"..\..\..\";
28+
return CodeConfiguration._root;
29+
}
30+
}
31+
32+
public static string NestFolder { get; } = $@"{Root}..\..\..\src\Nest\";
33+
public static string EsNetFolder { get; } = $@"{Root}..\..\..\src\Elasticsearch.Net\";
34+
public static string ViewFolder { get; } = $@"{Root}Views\";
35+
public static string RestSpecificationFolder { get; } = $@"{Root}RestSpecification\";
36+
37+
public static readonly Dictionary<string, string> MethodNameOverrides =
38+
(from f in new DirectoryInfo(NestFolder).GetFiles("*.cs", SearchOption.AllDirectories)
39+
let contents = File.ReadAllText(f.FullName)
40+
let c = Regex.Replace(contents, @"^.+\[DescriptorFor\(""([^ \r\n]+)""\)\].*$", "$1", RegexOptions.Singleline)
41+
where !c.Contains(" ") //filter results that did not match
42+
select new { Value = f.Name.Replace("Request", ""), Key = c })
43+
.DistinctBy(v => v.Key)
44+
.ToDictionary(k => k.Key, v => v.Value.Replace(".cs", ""));
45+
46+
public static readonly Dictionary<string, string> KnownDescriptors =
47+
(from f in new DirectoryInfo(NestFolder).GetFiles("*Request.cs", SearchOption.AllDirectories)
48+
let contents = File.ReadAllText(f.FullName)
49+
let c = Regex.Replace(contents, @"^.+class ([^ \r\n]+Descriptor(?:<[^>\r\n]+>)?[^ \r\n]*).*$", "$1", RegexOptions.Singleline)
50+
select new { Key = Regex.Replace(c, "<.*$", ""), Value = Regex.Replace(c, @"^.*?(?:(\<.+>).*?)?$", "$1") })
51+
.DistinctBy(v => v.Key)
52+
.OrderBy(v => v.Key)
53+
.ToDictionary(k => k.Key, v => v.Value);
54+
55+
public static readonly Dictionary<string, string> KnownRequests =
56+
(from f in new DirectoryInfo(NestFolder).GetFiles("*Request.cs", SearchOption.AllDirectories)
57+
let contents = File.ReadAllText(f.FullName)
58+
let c = Regex.Replace(contents, @"^.+interface ([^ \r\n]+Request(?:<[^>\r\n]+>)?[^ \r\n]*).*$", "$1", RegexOptions.Singleline)
59+
where c.StartsWith("I") && c.Contains("Request")
60+
select new { Key = Regex.Replace(c, "<.*$", ""), Value = Regex.Replace(c, @"^.*?(?:(\<.+>).*?)?$", "$1") })
61+
.DistinctBy(v => v.Key)
62+
.ToDictionary(k => k.Key, v => v.Value);
63+
64+
}
65+
}

‎src/CodeGeneration/CodeGeneration.LowLevelClient/RestSpecDownloader.cs

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
using System;
2+
using System.Collections.Generic;
3+
using System.IO;
4+
using System.Linq;
5+
using System.Net;
6+
using CsQuery;
7+
using ShellProgressBar;
8+
9+
namespace CodeGeneration.LowLevelClient
10+
{
11+
public class RestSpecDownloader
12+
{
13+
public static RestSpecDownloader Download(string branch) => new RestSpecDownloader(branch);
14+
15+
private static readonly Dictionary<string, string> OnlineSpecifications = new Dictionary<string, string>
16+
{
17+
{ "Core", "https://github.com/elastic/elasticsearch/tree/{version}/rest-api-spec/src/main/resources/rest-api-spec/api" },
18+
{ "DeleteByQuery", "https://github.com/elastic/elasticsearch/tree/{version}/plugins/delete-by-query/src/test/resources/rest-api-spec/api" },
19+
};
20+
private class Specification
21+
{
22+
public string FolderOnDisk { get; set; }
23+
public string GithubListingUrl { get; set; }
24+
public string Branch { get; set; }
25+
public string GithubDownloadUrl(string file) => this.GithubListingUrl.Replace("github.com", "raw.githubusercontent.com").Replace("tree/", "") + "/" + file;
26+
}
27+
28+
private static readonly ProgressBarOptions MainProgressBarOptions = new ProgressBarOptions { BackgroundColor = ConsoleColor.DarkGray };
29+
private static readonly ProgressBarOptions SubProgressBarOptions = new ProgressBarOptions
30+
{
31+
ForeGroundColor = ConsoleColor.Cyan,
32+
ForeGroundColorDone = ConsoleColor.DarkGreen,
33+
ProgressCharacter = '─',
34+
BackgroundColor = ConsoleColor.DarkGray,
35+
};
36+
37+
private RestSpecDownloader(string branch)
38+
{
39+
var specifications =
40+
(from kv in OnlineSpecifications
41+
let url = kv.Value.Replace("{version}", branch)
42+
select new Specification { FolderOnDisk = kv.Key, Branch = branch, GithubListingUrl = url }).ToList();
43+
44+
45+
using (var pbar = new ProgressBar(specifications.Count, "Downloading specifications", MainProgressBarOptions))
46+
{
47+
foreach (var spec in specifications)
48+
{
49+
pbar.UpdateMessage($"Downloading rest-api-spec to {spec.FolderOnDisk} for branch {branch}");
50+
DownloadJsonDefinitions(spec, pbar);
51+
pbar.Tick($"Downloaded rest-api-spec to {spec.FolderOnDisk} for branch {branch}");
52+
}
53+
}
54+
}
55+
56+
private void DownloadJsonDefinitions(Specification spec, IProgressBar pbar)
57+
{
58+
using (var client = new WebClient())
59+
{
60+
var html = client.DownloadString(spec.GithubListingUrl);
61+
FindJsonFilesOnListing(spec, html, pbar);
62+
}
63+
}
64+
65+
private void FindJsonFilesOnListing(Specification spec, string html, IProgressBar pbar)
66+
{
67+
if (!Directory.Exists(CodeConfiguration.RestSpecificationFolder)) Directory.CreateDirectory(CodeConfiguration.RestSpecificationFolder);
68+
69+
var dom = CQ.Create(html);
70+
71+
WriteToEndpointsFolder(spec.FolderOnDisk, "root.html", html);
72+
73+
var endpoints = dom[".js-navigation-open"]
74+
.Select(s => s.InnerText)
75+
.Where(s => !string.IsNullOrEmpty(s) && s.EndsWith(".json"))
76+
.ToList();
77+
78+
using (var subBar = pbar.Spawn(endpoints.Count, "fetching individual json files", SubProgressBarOptions))
79+
endpoints.ForEach(s => WriteEndpointFile(spec, s, subBar));
80+
}
81+
82+
private void WriteEndpointFile(Specification spec, string s, IProgressBar pbar)
83+
{
84+
var rawFile = spec.GithubDownloadUrl(s);
85+
using (var client = new WebClient())
86+
{
87+
var fileName = rawFile.Split('/').Last();
88+
var json = client.DownloadString(rawFile);
89+
WriteToEndpointsFolder(spec.FolderOnDisk, fileName, json);
90+
pbar.Tick($"Downloading {fileName}");
91+
}
92+
}
93+
94+
private void WriteToEndpointsFolder(string folder, string filename, string contents)
95+
{
96+
var f = Path.Combine(CodeConfiguration.RestSpecificationFolder, folder);
97+
if (!Directory.Exists(f)) Directory.CreateDirectory(f);
98+
File.WriteAllText(f + "\\" + filename, contents);
99+
}
100+
}
101+
}

‎src/CodeGeneration/CodeGeneration.LowLevelClient/ApiEndpoints/reindex.json renamed to ‎src/CodeGeneration/CodeGeneration.LowLevelClient/RestSpecification/Core/reindex.json

File renamed without changes.

‎src/CodeGeneration/CodeGeneration.LowLevelClient/ApiEndpoints/tasks.cancel.json renamed to ‎src/CodeGeneration/CodeGeneration.LowLevelClient/RestSpecification/Core/tasks.cancel.json

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88
"parts": {
99
"task_id": {
1010
"type": "string",
11-
"description": "Cancel the task with specified id"
11+
"description": "Cancel the task with specified task id (node_id:task_number)"
1212
}
1313
},
1414
"params": {
@@ -26,7 +26,7 @@
2626
},
2727
"parent_task": {
2828
"type" : "string",
29-
"description" : "Cancel tasks with specified parent task id. Set to -1 to cancel all."
29+
"description" : "Cancel tasks with specified parent task id (node_id:task_number). Set to -1 to cancel all."
3030
}
3131
}
3232
},

‎src/CodeGeneration/CodeGeneration.LowLevelClient/ApiEndpoints/tasks.list.json renamed to ‎src/CodeGeneration/CodeGeneration.LowLevelClient/RestSpecification/Core/tasks.list.json

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88
"parts": {
99
"task_id": {
1010
"type": "string",
11-
"description": "Return the task with specified id"
11+
"description": "Return the task with specified id (node_id:task_number)"
1212
}
1313
},
1414
"params": {
@@ -30,7 +30,7 @@
3030
},
3131
"parent_task": {
3232
"type" : "string",
33-
"description" : "Return tasks with specified parent task id. Set to -1 to return all."
33+
"description" : "Return tasks with specified parent task id (node_id:task_number). Set to -1 to return all."
3434
},
3535
"wait_for_completion": {
3636
"type": "boolean",

‎src/CodeGeneration/CodeGeneration.LowLevelClient/ApiEndpoints/update_by_query.json renamed to ‎src/CodeGeneration/CodeGeneration.LowLevelClient/RestSpecification/Core/update_by_query.json

File renamed without changes.

0 commit comments

Comments
 (0)