Skip to content

Commit 92d17b4

Browse files
svekarssvekars
authored
Merge branch 'main' into add-compiler-left-nav
2 parents 636061a + 1cedb08 commit 92d17b4

File tree

70 files changed

+2534
-1090
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

70 files changed

+2534
-1090
lines changed

‎.ci/docker/requirements.txt

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ tqdm==4.66.1
1414
numpy==1.24.4
1515
matplotlib
1616
librosa
17-
torch==2.5
17+
torch==2.6
1818
torchvision
1919
torchdata
2020
networkx
@@ -28,8 +28,8 @@ tensorboard
2828
jinja2==3.1.3
2929
pytorch-lightning
3030
torchx
31-
torchrl==0.5.0
32-
tensordict==0.5.0
31+
torchrl==0.6.0
32+
tensordict==0.6.0
3333
ax-platform>=0.4.0
3434
nbformat>=5.9.2
3535
datasets
@@ -70,4 +70,4 @@ semilearn==0.3.2
7070
torchao==0.5.0
7171
segment_anything==1.0
7272
torchrec==1.0.0; platform_system == "Linux"
73-
fbgemm-gpu==1.0.0; platform_system == "Linux"
73+
fbgemm-gpu==1.1.0; platform_system == "Linux"

‎.github/workflows/build-tutorials.yml

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,8 @@ jobs:
4444
4545
- name: Checkout Tutorials
4646
uses: actions/checkout@v3
47+
with:
48+
fetch-depth: 0
4749

4850
- name: Setup Linux
4951
uses: pytorch/pytorch/.github/actions/setup-linux@main
@@ -115,6 +117,8 @@ jobs:
115117
116118
- name: Checkout Tutorials
117119
uses: actions/checkout@v3
120+
with:
121+
fetch-depth: 0
118122

119123
- name: Setup Linux
120124
uses: pytorch/pytorch/.github/actions/setup-linux@main

‎.github/workflows/spelling.yml

Lines changed: 137 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5,16 +5,149 @@ on:
55
push:
66
branches:
77
- main
8+
89
jobs:
910
pyspelling:
1011
runs-on: ubuntu-20.04
1112
steps:
12-
- uses: actions/checkout@v3
13+
- name: Check for skip label and get changed files
14+
id: check-files
15+
uses: actions/github-script@v6
16+
with:
17+
script: |
18+
let skipCheck = false;
19+
let changedFiles = [];
20+
21+
if (context.eventName === 'pull_request') {
22+
// Check for skip label
23+
const { data: labels } = await github.rest.issues.listLabelsOnIssue({
24+
owner: context.repo.owner,
25+
repo: context.repo.repo,
26+
issue_number: context.issue.number
27+
});
28+
skipCheck = labels.some(label => label.name === 'skip-spell-check');
29+
30+
if (!skipCheck) {
31+
// Get changed files in PR
32+
const { data: files } = await github.rest.pulls.listFiles({
33+
owner: context.repo.owner,
34+
repo: context.repo.repo,
35+
pull_number: context.issue.number
36+
});
37+
38+
changedFiles = files
39+
.filter(file => file.filename.match(/\.(py|rst|md)$/))
40+
.map(file => file.filename);
41+
}
42+
} else {
43+
// For push events, we'll still need to use git diff
44+
// We'll handle this after checkout
45+
}
46+
47+
core.setOutput('skip', skipCheck.toString());
48+
core.setOutput('files', changedFiles.join('\n'));
49+
core.setOutput('is-pr', (context.eventName === 'pull_request').toString());
50+
51+
- uses: actions/checkout@v4
52+
if: steps.check-files.outputs.skip != 'true'
53+
with:
54+
fetch-depth: 0
55+
56+
- name: Get changed files for push event
57+
if: |
58+
steps.check-files.outputs.skip != 'true' &&
59+
steps.check-files.outputs.is-pr != 'true'
60+
id: push-files
61+
run: |
62+
CHANGED_FILES=$(git diff --name-only HEAD^..HEAD -- '*.py' '*.rst' '*.md')
63+
echo "files<<EOF" >> $GITHUB_OUTPUT
64+
echo "$CHANGED_FILES" >> $GITHUB_OUTPUT
65+
echo "EOF" >> $GITHUB_OUTPUT
66+
67+
- name: Check if relevant files changed
68+
if: steps.check-files.outputs.skip != 'true'
69+
id: check
70+
run: |
71+
if [ "${{ steps.check-files.outputs.is-pr }}" == "true" ]; then
72+
FILES="${{ steps.check-files.outputs.files }}"
73+
else
74+
FILES="${{ steps.push-files.outputs.files }}"
75+
fi
76+
77+
if [ -z "$FILES" ]; then
78+
echo "skip=true" >> $GITHUB_OUTPUT
79+
echo "No relevant files changed (*.py, *.rst, *.md), skipping spell check"
80+
else
81+
echo "skip=false" >> $GITHUB_OUTPUT
82+
echo "Found changed files to check:"
83+
echo "$FILES"
84+
fi
85+
1386
- uses: actions/setup-python@v4
87+
if: |
88+
steps.check-files.outputs.skip != 'true' &&
89+
steps.check.outputs.skip != 'true'
1490
with:
1591
python-version: '3.9'
1692
cache: 'pip'
17-
- run: pip install pyspelling
18-
- run: sudo apt-get install aspell aspell-en
19-
- run: pyspelling
2093

94+
- name: Install dependencies
95+
if: |
96+
steps.check-files.outputs.skip != 'true' &&
97+
steps.check.outputs.skip != 'true'
98+
run: |
99+
pip install pyspelling
100+
sudo apt-get install aspell aspell-en
101+
102+
- name: Run spell check on each file
103+
id: spellcheck
104+
if: |
105+
steps.check-files.outputs.skip != 'true' &&
106+
steps.check.outputs.skip != 'true'
107+
run: |
108+
if [ "${{ steps.check-files.outputs.is-pr }}" == "true" ]; then
109+
mapfile -t FILES <<< "${{ steps.check-files.outputs.files }}"
110+
else
111+
mapfile -t FILES <<< "${{ steps.push-files.outputs.files }}"
112+
fi
113+
114+
# Check each file individually
115+
FINAL_EXIT_CODE=0
116+
SPELLCHECK_LOG=""
117+
for file in "${FILES[@]}"; do
118+
if [ -n "$file" ]; then
119+
echo "Checking spelling in $file"
120+
python3 -c "import yaml; config = yaml.safe_load(open('.pyspelling.yml')); new_matrix = [matrix.copy() for matrix in config['matrix'] if (('python' in matrix['name'].lower() and '$file'.endswith('.py')) or ('rest' in matrix['name'].lower() and '$file'.endswith('.rst')) or ('markdown' in matrix['name'].lower() and '$file'.endswith('.md'))) and not matrix.update({'sources': ['$file']})]; config['matrix'] = new_matrix; yaml.dump(config, open('temp_config.yml', 'w'))"
121+
122+
if OUTPUT=$(pyspelling -c temp_config.yml 2>&1); then
123+
echo "No spelling errors found in $file"
124+
else
125+
FINAL_EXIT_CODE=1
126+
echo "Spelling errors found in $file:"
127+
echo "$OUTPUT"
128+
SPELLCHECK_LOG+="### $file\n$OUTPUT\n\n"
129+
fi
130+
fi
131+
done
132+
133+
# Save the results to GITHUB_OUTPUT
134+
echo "spell_failed=$FINAL_EXIT_CODE" >> $GITHUB_OUTPUT
135+
echo "spell_log<<SPELLEOF" >> $GITHUB_OUTPUT
136+
echo "$SPELLCHECK_LOG" >> $GITHUB_OUTPUT
137+
echo "SPELLEOF" >> $GITHUB_OUTPUT
138+
139+
if [ $FINAL_EXIT_CODE -ne 0 ]; then
140+
echo "Spell check failed! See above for details."
141+
echo
142+
echo "Here are a few tips:"
143+
echo "- All PyTorch API objects must be in double backticks or use an intersphinx directive."
144+
echo " Example: ``torch.nn``, :func:"
145+
echo "- Consult en-wordlist.txt for spellings of some of the words."
146+
echo " You can add a word to en-wordlist.txt if:"
147+
echo " 1) It's a common abbreviation, like RNN."
148+
echo " 2) It's a word widely accepted in the industry."
149+
echo "- Please do not add words like 'dtype', 'torch.nn.Transformer' to pass spellcheck."
150+
echo " Instead wrap it in double backticks or use an intersphinx directive."
151+
echo
152+
exit 1
153+
fi

‎.jenkins/build.sh

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,8 +22,10 @@ sudo apt-get install -y pandoc
2222
#Install PyTorch Nightly for test.
2323
# Nightly - pip install --pre torch torchvision torchaudio -f https://download.pytorch.org/whl/nightly/cu102/torch_nightly.html
2424
# Install 2.5 to merge all 2.4 PRs - uncomment to install nightly binaries (update the version as needed).
25-
# pip uninstall -y torch torchvision torchaudio torchtext torchdata
26-
# pip3 install torch==2.5.0 torchvision torchaudio --no-cache-dir --index-url https://download.pytorch.org/whl/test/cu124
25+
# sudo pip uninstall -y torch torchvision torchaudio torchtext torchdata
26+
# sudo pip3 install torch==2.6.0 torchvision --no-cache-dir --index-url https://download.pytorch.org/whl/test/cu124
27+
# sudo pip uninstall -y fbgemm-gpu torchrec
28+
# sudo pip3 install fbgemm-gpu==1.1.0 torchrec==1.0.0 --no-cache-dir --index-url https://download.pytorch.org/whl/test/cu124
2729

2830
# Install two language tokenizers for Translation with TorchText tutorial
2931
python -m spacy download en_core_web_sm

‎.jenkins/insert_last_verified.py

Lines changed: 160 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,160 @@
1+
import json
2+
import os
3+
import subprocess
4+
import sys
5+
from datetime import datetime
6+
7+
from bs4 import BeautifulSoup
8+
9+
10+
json_file_path = "tutorials-review-data.json"
11+
12+
# paths to skip from the post-processing script
13+
paths_to_skip = [
14+
"beginner/examples_autograd/two_layer_net_custom_function", # not present in the repo
15+
"beginner/examples_nn/two_layer_net_module", # not present in the repo
16+
"beginner/examples_tensor/two_layer_net_numpy", # not present in the repo
17+
"beginner/examples_tensor/two_layer_net_tensor", # not present in the repo
18+
"beginner/examples_autograd/two_layer_net_autograd", # not present in the repo
19+
"beginner/examples_nn/two_layer_net_optim", # not present in the repo
20+
"beginner/examples_nn/two_layer_net_nn", # not present in the repo
21+
"intermediate/coding_ddpg", # not present in the repo - will delete the carryover
22+
]
23+
# Mapping of source directories to build directories
24+
source_to_build_mapping = {
25+
"beginner": "beginner_source",
26+
"recipes": "recipes_source",
27+
"distributed": "distributed",
28+
"intermediate": "intermediate_source",
29+
"prototype": "prototype_source",
30+
"advanced": "advanced_source",
31+
"": "", # root dir for index.rst
32+
}
33+
34+
def get_git_log_date(file_path, git_log_args):
35+
try:
36+
result = subprocess.run(
37+
["git", "log"] + git_log_args + ["--", file_path],
38+
capture_output=True,
39+
text=True,
40+
check=True,
41+
)
42+
if result.stdout:
43+
date_str = result.stdout.splitlines()[0]
44+
return datetime.strptime(date_str, "%a, %d %b %Y %H:%M:%S %z")
45+
except subprocess.CalledProcessError:
46+
pass
47+
raise ValueError(f"Could not find date for {file_path}")
48+
49+
def get_creation_date(file_path):
50+
return get_git_log_date(file_path, ["--diff-filter=A", "--format=%aD"]).strftime("%b %d, %Y")
51+
52+
53+
def get_last_updated_date(file_path):
54+
return get_git_log_date(file_path, ["-1", "--format=%aD"]).strftime("%b %d, %Y")
55+
56+
# Try to find the source file with the given base path and the extensions .rst and .py
57+
def find_source_file(base_path):
58+
for ext in [".rst", ".py"]:
59+
source_file_path = base_path + ext
60+
if os.path.exists(source_file_path):
61+
return source_file_path
62+
return None
63+
64+
65+
# Function to process a JSON file and insert the "Last Verified" information into the HTML files
66+
def process_json_file(build_dir , json_file_path):
67+
with open(json_file_path, "r", encoding="utf-8") as json_file:
68+
json_data = json.load(json_file)
69+
70+
for entry in json_data:
71+
path = entry["Path"]
72+
last_verified = entry["Last Verified"]
73+
status = entry.get("Status", "")
74+
if path in paths_to_skip:
75+
print(f"Skipping path: {path}")
76+
continue
77+
if status in ["needs update", "not verified"]:
78+
formatted_last_verified = "Not Verified"
79+
elif last_verified:
80+
try:
81+
last_verified_date = datetime.strptime(last_verified, "%Y-%m-%d")
82+
formatted_last_verified = last_verified_date.strftime("%b %d, %Y")
83+
except ValueError:
84+
formatted_last_verified = "Unknown"
85+
else:
86+
formatted_last_verified = "Not Verified"
87+
if status == "deprecated":
88+
formatted_last_verified += "Deprecated"
89+
90+
for build_subdir, source_subdir in source_to_build_mapping.items():
91+
if path.startswith(build_subdir):
92+
html_file_path = os.path.join(build_dir, path + ".html")
93+
base_source_path = os.path.join(
94+
source_subdir, path[len(build_subdir) + 1 :]
95+
)
96+
source_file_path = find_source_file(base_source_path)
97+
break
98+
else:
99+
print(f"Warning: No mapping found for path {path}")
100+
continue
101+
102+
if not os.path.exists(html_file_path):
103+
print(
104+
f"Warning: HTML file not found for path {html_file_path}."
105+
"If this is a new tutorial, please add it to the audit JSON file and set the Verified status and todays's date."
106+
)
107+
continue
108+
109+
if not source_file_path:
110+
print(f"Warning: Source file not found for path {base_source_path}.")
111+
continue
112+
113+
created_on = get_creation_date(source_file_path)
114+
last_updated = get_last_updated_date(source_file_path)
115+
116+
with open(html_file_path, "r", encoding="utf-8") as file:
117+
soup = BeautifulSoup(file, "html.parser")
118+
# Check if the <p> tag with class "date-info-last-verified" already exists
119+
existing_date_info = soup.find("p", {"class": "date-info-last-verified"})
120+
if existing_date_info:
121+
print(
122+
f"Warning: <p> tag with class 'date-info-last-verified' already exists in {html_file_path}"
123+
)
124+
continue
125+
126+
h1_tag = soup.find("h1") # Find the h1 tag to insert the dates
127+
if h1_tag:
128+
date_info_tag = soup.new_tag("p", **{"class": "date-info-last-verified"})
129+
date_info_tag["style"] = "color: #6c6c6d; font-size: small;"
130+
# Add the "Created On", "Last Updated", and "Last Verified" information
131+
date_info_tag.string = (
132+
f"Created On: {created_on} | "
133+
f"Last Updated: {last_updated} | "
134+
f"Last Verified: {formatted_last_verified}"
135+
)
136+
# Insert the new tag after the <h1> tag
137+
h1_tag.insert_after(date_info_tag)
138+
# Save back to the HTML.
139+
with open(html_file_path, "w", encoding="utf-8") as file:
140+
file.write(str(soup))
141+
else:
142+
print(f"Warning: <h1> tag not found in {html_file_path}")
143+
144+
145+
def main():
146+
if len(sys.argv) < 2:
147+
print("Error: Build directory not provided. Exiting.")
148+
exit(1)
149+
build_dir = sys.argv[1]
150+
print(f"Build directory: {build_dir}")
151+
process_json_file(build_dir , json_file_path)
152+
print(
153+
"Finished processing JSON file. Please check the output for any warnings. "
154+
"Pages like `nlp/index.html` are generated only during the full `make docs` "
155+
"or `make html` build. Warnings about these files when you run `make html-noplot` "
156+
"can be ignored."
157+
)
158+
159+
if __name__ == "__main__":
160+
main()

‎.jenkins/validate_tutorials_built.py

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,6 @@
2525
"intermediate_source/mnist_train_nas", # used by ax_multiobjective_nas_tutorial.py
2626
"intermediate_source/fx_conv_bn_fuser",
2727
"intermediate_source/_torch_export_nightly_tutorial", # does not work on release
28-
"intermediate_source/transformer_building_blocks", # does not work on release
2928
"advanced_source/super_resolution_with_onnxruntime",
3029
"advanced_source/usb_semisup_learn", # fails with CUDA OOM error, should try on a different worker
3130
"prototype_source/fx_graph_mode_ptq_dynamic",
@@ -51,7 +50,6 @@
5150
"intermediate_source/flask_rest_api_tutorial",
5251
"intermediate_source/text_to_speech_with_torchaudio",
5352
"intermediate_source/tensorboard_profiler_tutorial", # reenable after 2.0 release.
54-
"intermediate_source/torch_export_tutorial" # reenable after 2940 is fixed.
5553
]
5654

5755
def tutorial_source_dirs() -> List[Path]:

‎.lycheeignore

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,3 +12,9 @@ https://pytorch.org/tutorials/beginner/colab/n
1212

1313
# Ignore local host link from intermediate_source/tensorboard_tutorial.rst
1414
http://localhost:6006
15+
16+
# Ignore local host link from recipes_source/deployment_with_flask.rst
17+
http://localhost:5000/predict
18+
19+
# Ignore local host link from advanced_source/cpp_frontend.rst
20+
https://www.uber.com/blog/deep-neuroevolution/

0 commit comments

Comments
 (0)