Skip to content

Commit 9c23834

Browse files
yosifkityosifkit
committed
Add repo metadata, start with categories!
scripts for checking repo categories, updating the canonical set added categories to push.pl
1 parent 1e39fa8 commit 9c23834

File tree

7 files changed

+237
-0
lines changed

7 files changed

+237
-0
lines changed

‎.ci/check-metadata.sh

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
#!/usr/bin/env bash
2+
set -Eeuo pipefail
3+
4+
cd "$(dirname "$(readlink -f "$BASH_SOURCE")")/.."
5+
6+
# metadata.sh takes directories with a 'metadata.json' in them
7+
# metadata.json is expected in every repo
8+
# "." so that the canonical source metadata.json is checked too
9+
./metadata.sh -d -c */ .

‎.github/workflows/ci.yml

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,3 +46,8 @@ jobs:
4646
fetch-depth: 0
4747
- run: .ci/check-pr-no-readme.sh
4848
if: ${{ github.event_name == 'pull_request' }}
49+
metadata:
50+
runs-on: ubuntu-latest
51+
steps:
52+
- uses: actions/checkout@v3
53+
- run: .ci/check-metadata.sh

‎README.md

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,7 @@ After opening your Pull Request the changes will be checked by an automated `mar
6161
- Create a `license.md` (required)
6262
- Create a `maintainer.md` (required)
6363
- Create a `github-repo` (required)
64+
- Create a `metadata.json` (required)
6465
- Add a `logo.png` (recommended)
6566

6667
Optionally:
@@ -128,6 +129,18 @@ The image is automatically scaled to a 120 pixel square for the top of the Docke
128129

129130
This file should contain a link to the maintainers of the Dockerfile.
130131

132+
## `metadata.json`
133+
134+
This file contains data about the repo for Docker Hub. The minimum file is defined below. `./metadata.sh [repo-name]` must be used to correctly format it (use `-f` to apply its suggested changes). There is a limit to the number of Docker Hub categories allowed; run with `-c` to check the limit and categories. `metadata.json` in the root contains the list of categories to choose from.
135+
136+
```json
137+
{
138+
"hub": {
139+
"categories": []
140+
}
141+
}
142+
```
143+
131144
## `README-short.txt`
132145

133146
This is the short description for the Docker Hub, limited to 100 characters in a single line.

‎get-categories.sh

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
#!/usr/bin/env bash
2+
set -Eeuo pipefail
3+
4+
workdir="$(readlink -f "$BASH_SOURCE")"
5+
workdir="$(dirname "$workdir")"
6+
7+
jsonFile='metadata.json'
8+
canonicalMetadataFile="$workdir/$jsonFile"
9+
10+
allCategories="$(curl -fsSL https://hub.docker.com/v2/categories)"
11+
export allCategories
12+
13+
# add categories slugs to canonicalMetadataFile without losing other keys there
14+
json="$(
15+
jq --sort-keys '
16+
(env.allCategories | fromjson) as $allCategories
17+
| .hub.categories = ( [ $allCategories[].slug ] | sort )
18+
' "$canonicalMetadataFile"
19+
)"
20+
21+
echo "$json" | tee "$canonicalMetadataFile"

‎metadata.json

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
{
2+
"hub": {
3+
"categories": [
4+
"api-management",
5+
"content-management-system",
6+
"data-science",
7+
"databases-and-storage",
8+
"developer-tools",
9+
"integration-and-delivery",
10+
"internet-of-things",
11+
"languages-and-frameworks",
12+
"machine-learning-and-ai",
13+
"message-queues",
14+
"monitoring-and-observability",
15+
"networking",
16+
"operating-systems",
17+
"security",
18+
"web-analytics",
19+
"web-servers"
20+
]
21+
}
22+
}

‎metadata.sh

Lines changed: 140 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,140 @@
1+
#!/usr/bin/env bash
2+
set -Eeuo pipefail
3+
4+
workdir="$(readlink -f "$BASH_SOURCE")"
5+
workdir="$(dirname "$workdir")"
6+
#cd "$workdir"
7+
8+
jsonFile='metadata.json'
9+
canonicalMetadataFile="$workdir/$jsonFile"
10+
maxCategories=3
11+
12+
self="$(basename "$0")"
13+
14+
usage() {
15+
cat <<EOUSAGE
16+
17+
usage: $self [--categories] [--diff] [--fmt] repo [...]
18+
ie: $self debian
19+
$self -d */
20+
$self -dc python
21+
22+
This script checks a repo's metadata.json and checks categories, diffs, or formats it.
23+
24+
-c, --categories Check that the categories are from the valid set of categories and that there are no more than $maxCategories. Exits non-zero if there are too many categories or they are unkown.
25+
-d, --diff Check formatting of the '[repo]/metadata.json' and print a diff. Default action if no flags are supplied. Exits non-zero if there is a difference.
26+
-f, --fmt, --format Apply the formatting that the '-d' flag would output.
27+
-h, --help Print this help output and exit.
28+
29+
Arguments are the list of repos with a 'metadata.json' in them. 'metadata.json' is expected in every repo
30+
'.' can also be passed to check the format of the canonical './metadata.json' at
31+
the root of the repo, but the max categories of '-c' is skipped for it.
32+
EOUSAGE
33+
}
34+
35+
# arg handling
36+
opts="$(getopt -o 'cdfh' --long 'categories,diff,fmt,format,help' -- "$@" || { usage >&2 && false; })"
37+
eval set -- "$opts"
38+
39+
categories=
40+
diff=
41+
fmt=
42+
43+
while true; do
44+
flag="$1"
45+
shift
46+
case "$flag" in
47+
--categories|-c) categories=1 ;;
48+
--diff|-d) diff=1 ;;
49+
--fmt|--format|-f) fmt=1 ;;
50+
--help|-h) usage && exit 0 ;;
51+
--) break ;;
52+
*)
53+
{
54+
echo "error: unknown flag: $flag"
55+
usage
56+
} >&2
57+
exit 1
58+
;;
59+
esac
60+
done
61+
62+
# default to print diff
63+
if [ -z "$diff" ] && [ -z "$categories" ] && [ -z "$fmt" ]; then
64+
diff=1
65+
fi
66+
67+
repos=( "$@" )
68+
if [ "${#repos[@]}" -eq 0 ]; then
69+
repos=( */ )
70+
fi
71+
repos=( "${repos[@]%/}" )
72+
73+
canonicalMetadataFileJson="$(cat "$canonicalMetadataFile")"
74+
export canonicalMetadataFileJson
75+
76+
failure=
77+
for repo in "${repos[@]}"; do
78+
repoFile="$workdir/$repo/$jsonFile"
79+
if [ ! -f "$repoFile" ]; then
80+
failure=1
81+
echo "error file does not exist: $repo/$jsonFile"
82+
continue
83+
fi
84+
repoFile="$(readlink -f "$repoFile")"
85+
86+
if [ -n "$diff" ] || [ -n "$fmt" ]; then
87+
# sort object keys and pretty print with jq as our "cannonical json"
88+
# sort categories array
89+
if ! repoFilejson="$(jq --sort-keys '.hub.categories |= sort' "$repoFile")"; then
90+
echo "error $repo/$jsonFile improper json"
91+
failure=1
92+
continue
93+
fi
94+
if ! filediff="$(diff -u "$repoFile" <(echo "$repoFilejson"))"; then
95+
if [ -n "$diff" ]; then
96+
echo >&2 "$repoFile"
97+
# skip diff headers
98+
echo "$filediff" | tail -n +3
99+
failure=1
100+
fi
101+
if [ -n "$fmt" ]; then
102+
# TODO should fmt + categories remove invalid or categories over max?
103+
echo "$repoFilejson" > "$repoFile"
104+
fi
105+
fi
106+
fi
107+
108+
# TODO also check for required keys and/or types?
109+
if [ -n "$categories" ]; then
110+
# the canonicalMetadataFile doesn't have too many categories since it is the source of categories
111+
# all other metadata.json files must not be more than maxCategories
112+
if [ "$canonicalMetadataFile" != "$repoFile" ]; then
113+
tooManyCategories="$(jq --raw-output '
114+
.hub.categories
115+
| if length > '"$maxCategories"' then
116+
length
117+
else empty end
118+
' "$repoFile")"
119+
if [ -n "$tooManyCategories" ]; then
120+
echo >&2 "$repo, error too many Docker Hub categories: $tooManyCategories, max $maxCategories"
121+
failure=1
122+
fi
123+
fi
124+
# check for categories that aren't in the canonical set
125+
extraCategories="$(jq -c '
126+
(env.canonicalMetadataFileJson | fromjson | .hub.categories) as $canonicalCategories
127+
| .hub.categories
128+
| map(. as $cat | select($canonicalCategories | index($cat) < 0))
129+
| if length > 0 then . else empty end
130+
' "$repoFile")"
131+
if [ -n "$extraCategories" ]; then
132+
echo >&2 "$repo, error unkown categories: $extraCategories"
133+
failure=1
134+
fi
135+
fi
136+
done
137+
138+
if [ "$failure" ]; then
139+
exit 1
140+
fi

‎push.pl

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
use File::Temp;
99
use Getopt::Long;
1010
use Mojo::File;
11+
use Mojo::JSON qw(decode_json);
1112
use Mojo::UserAgent;
1213
use Mojo::Util qw(decode encode trim url_escape);
1314

@@ -211,6 +212,32 @@ sub prompt_for_edit {
211212
my $repoDetails = $repoTx->res->json;
212213
$repoDetails->{description} //= '';
213214
$repoDetails->{full_description} //= '';
215+
$repoDetails->{categories} //= [];
216+
my @repoCategories = sort map { $_->{slug} } @{$repoDetails->{categories}};
217+
218+
# read local categories from metadata.json
219+
my $repoMetadataBytes = Mojo::File->new($repoName . '/metadata.json')->slurp;
220+
my $repoMetadataJson = decode_json $repoMetadataBytes;
221+
my @localRepoCategories = sort @{$repoMetadataJson->{hub}{categories}};
222+
223+
# check if the local categories differ in length or items from the remote
224+
my $needCat = @localRepoCategories != @repoCategories;
225+
if (! $needCat) {
226+
foreach my $i (0..@localRepoCategories) {
227+
last if ! defined $repoCategories[$i]; # length difference already covered, so we can bail
228+
if ($localRepoCategories[$i] ne $repoCategories[$i]) {
229+
$needCat = 1;
230+
last;
231+
}
232+
}
233+
}
234+
if ($needCat) {
235+
say 'updating ' . $repoName . ' categories';
236+
my $catsPatch = $ua->patch($repoUrl . 'categories/' => { %$authorizationHeader, Accept => 'application/json' } => json => [
237+
map { { slug => $_ => name => 'All those moments will be lost in time, like tears in rain... Time to die.' } } @{$repoMetadataJson->{hub}{categories}}
238+
]);
239+
die 'patch to categories failed: ' . $catsPatch->res->text unless $catsPatch->res->is_success;
240+
}
214241

215242
my $hubShort = prompt_for_edit($repoDetails->{description}, $repoName . '/README-short.txt');
216243
my $hubLong = prompt_for_edit($repoDetails->{full_description}, $repoName . '/README.md', $hubLengthLimit);

0 commit comments

Comments
 (0)