Export diff check results to a zip archive

.github/workflows/check_diff.yml

@@ -31,3 +31,9 @@ jobs:
 
     - name: check diff
       run: bash ${GITHUB_WORKSPACE}/ci/check_diff.sh ${{ github.event.inputs.clone_url }} ${{ github.event.inputs.branch_name }} ${{ github.event.inputs.commit_hash || github.event.inputs.branch_name }} ${{ github.event.inputs.rustfmt_configs }}
+
+    - name: Archive Diff Check Report
+      uses: actions/upload-artifact@v3
+      with:
+        name: diff-check-report
+        path: diff-check.zip

ci/check_diff.sh

@@ -71,6 +71,7 @@ function create_diff() {
 # $RUSFMT_BIN: Path to the rustfmt master binary. Created when running `compile_rustfmt`
 # $FEATURE_BIN: Path to the rustfmt feature binary. Created when running `compile_rustfmt`
 # $OPTIONAL_RUSTFMT_CONFIGS: Optional configs passed to the script from $4
+# $OUTPUT_DIR: Path to an output directory for storing the diff files. Set in `main`
 function check_diff() {
     echo "running rustfmt (master) on $1"
     create_diff $RUSFMT_BIN rustfmt_diff.txt
@@ -89,10 +90,18 @@ function check_diff() {
         --unified=0 --no-index rustfmt_diff.txt feature_diff.txt 2>&1 | tail -n +6 | cut -c 2-
     )
 
+    # COPY diffs into output dir
+    mkdir $OUTPUT_DIR/$1
+    echo "Copying diffs to $OUTPUT_DIR/$1"
+    cp rustfmt_diff.txt $OUTPUT_DIR/$1/rustfmt_diff.txt
+    cp feature_diff.txt $OUTPUT_DIR/$1/feature_diff.txt
+
     if [ -z "$diff" ]; then
         echo "no diff detected between rustfmt and the feture branch"
         return 0
     else
+        echo "Copying diffs between rustfmt and feature branch to $OUTPUT_DIR/$1/diff.txt"
+        echo "$diff" >> $OUTPUT_DIR/$1/diff.txt
         echo "$diff"
         return 1
     fi
@@ -114,15 +123,28 @@ function compile_rustfmt() {
     git remote add feature $REMOTE_REPO
     git fetch feature $FEATURE_BRANCH
 
-    cargo build --release --bin rustfmt && cp target/release/rustfmt $1/rustfmt
+    CARGO_VERSON=$(cargo --version)
+    echo -e "\ncompiling with $CARGO_VERSON\n"
+
+    echo "Building rustfmt from src"
+    cargo build -q --release --bin rustfmt && cp target/release/rustfmt $1/rustfmt
+
     if [ -z "$OPTIONAL_COMMIT_HASH" ] || [ "$FEATURE_BRANCH" = "$OPTIONAL_COMMIT_HASH" ]; then
         git switch $FEATURE_BRANCH
     else
         git switch $OPTIONAL_COMMIT_HASH --detach
     fi
-    cargo build --release --bin rustfmt && cp target/release/rustfmt $1/feature_rustfmt
+
+    echo "Building feature rustfmt from src"
+    cargo build -q --release --bin rustfmt && cp target/release/rustfmt $1/feature_rustfmt
+
     RUSFMT_BIN=$1/rustfmt
+    RUSTFMT_VERSION=$($RUSFMT_BIN --version)
+    echo -e "\nRUSFMT_BIN $RUSTFMT_VERSION\n"
+
     FEATURE_BIN=$1/feature_rustfmt
+    FEATURE_VERSION=$($FEATURE_BIN --version)
+    echo -e "FEATURE_BIN $FEATURE_VERSION\n"
 }
 
 # Check the diff for running rustfmt and the feature branch on all the .rs files in the repo.
@@ -155,16 +177,107 @@ function check_repo() {
     STATUSES+=($?)
     set -e
 
-    echo "removing tmp_dir $tmp_dir"
+    echo -e "removing tmp_dir $tmp_dir\n\n"
     rm -rf $tmp_dir
     cd $WORKDIR
 }
 
+function write_readme() {
+    rustfmt_diff=\`rustfmt_diff.txt\`
+    feature_diff=\`feature_diff.txt\`
+    diff_file=\`diff.txt\`
+    diff_files=\`*_diff.txt\`
+
+    if [ -n "$OPTIONAL_RUSTFMT_CONFIGS" ]; then
+        OPTIONAL_CONFIG_DETAILS="* diff check optional configs: \`$OPTIONAL_RUSTFMT_CONFIGS\`"
+    fi
+
+    cat > README.md << EOL
+# Diff Check
+
+## Summary
+
+The Diff Check Job is used to validate rustfmts backwards compatability guarantees
+by running the latest rustfmt from [rust-lang/rustfmt](https://github.com/rust-lang/rustfmt) and
+comparing the formatting results against a fork or feature branch of rustfmt --
+often before deciding to merge those changes into rustfmt via a pull request.
+
+**cargo details**
+* version: \`$CARGO_VERSON\`
+
+**rustfmt details**
+* version: \`$RUSTFMT_VERSION\`
+
+**fork details**
+* repo url: $REMOTE_REPO
+* feature branch: \`$FEATURE_BRANCH\`
+* version: \`$FEATURE_VERSION\`
+$OPTIONAL_CONFIG_DETAILS
+
+## How to interpret results
+
+Diffs created by running the rustfmt binary are reported in $rustfmt_diff, and
+diffs created by running the forked rustfmt binary are stored in $feature_diff.
+The presence of $rustfmt_diff and $feature_diff are not indicative of any errors.
+Some of the real world projects that rustfmt is tested against may not use rustfmt at all.
+All the $diff_files files show is that using rustfmt on a given project would change some formatting.
+
+If a $diff_file file is present for a given project then that indicates a failure to
+uphold rustfmts backwards compatability guarantees. Given the same input both binaries produced different outputs.
+The $diff_file shows the difference in formatting output between both binaries.
+
+## How to inspect diff-check results
+
+First unzip the the diff-check archive
+
+\`\`\`
+unzip diff-check.zip -d diff-check
+\`\`\`
+
+If the diff-check job completes successfully that means that both the rustfmt binary and the forked rustfmt binary
+agree upon formatting changes. However, if the job fails because both binaries produced different formatting, you
+can inspect the differences by running:
+
+\`\`\`
+for file in \$(find diff-check -type f -name diff.txt); cat \$file
+\`\`\`
+
+If you're curious you can inspect formatting changes produced when running rustfmt by running:
+
+\`\`\`
+for file in \$(find diff-check -type f -name rustfmt_diff.txt); cat \$file
+\`\`\`
+
+Similarly, you can inspect formatting changes produced when running the forked rustfmt binary by running:
+
+\`\`\`
+for file in \$(find diff-check -type f -name feature_diff.txt); cat \$file
+\`\`\`
+EOL
+}
+
+# Zip up all the diff changes detected by the script
+#
+# Globlas:
+# $OUTPUT_DIR: Output directory where all `*diif.txt` files are written to. Set in `main`.
+# $CURRENT_DIR: The directory where the script was run from. Set in `main`.
+function zip_up_diffs() {
+    cd $OUTPUT_DIR
+    write_readme
+
+    # Just to clean things up we'll make sure to remove empty files and directories
+    find . -type f -empty -delete
+    find . -type d -empty -delete
+    zip -q -r $CURRENT_DIR/diff-check .
+}
+
 function main() {
+    CURRENT_DIR=$(pwd)
     tmp_dir=$(mktemp -d -t rustfmt-XXXXXXXX)
     echo Created tmp_dir $tmp_dir
 
     compile_rustfmt $tmp_dir
+    OUTPUT_DIR=$(mktemp -d -t diff-output-XXX)
 
     # run checks
     check_repo "https://github.com/rust-lang/rust.git" rust-lang-rust
@@ -191,9 +304,12 @@ function main() {
     check_repo "https://github.com/actix/actix.git" actix
     check_repo "https://github.com/denoland/deno.git" denoland_deno
 
+    zip_up_diffs
+
     # cleanup temp dir
-    echo removing tmp_dir $tmp_dir
+    echo removing tmp_dir $tmp_dir and $OUTPUT_DIR
     rm -rf $tmp_dir
+    rm -rf $OUTPUT_DIR
 
     # figure out the exit code
     for status in ${STATUSES[@]}