Restore content validation script

.github/workflows/workflow-validate.yaml

@@ -0,0 +1,36 @@
+name: Validate Content
+
+on:
+  repository_dispatch:
+    types: deploy
+  push:
+    branches:
+      - main
+  pull_request:
+    types: [opened, synchronize, ready_for_review, reopened]
+
+jobs:
+  spell_check:
+    name: Spell Check
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v1
+      - name: Run spell check
+        run: |
+          pip install codespell
+          codespell -I "scripts/resources/spell-check-ignore-list.txt" --skip="*.svg,*.dxf" ./content/
+
+  lint:
+    name: Content Linter
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        node_version: [14]
+
+    steps:
+      - uses: actions/checkout@v1
+
+      - name: Run linter
+        run: |
+          cd scripts/validation
+          ./content-lint.sh -p '../../content/hardware/'

.gitignore

@@ -6,5 +6,4 @@ envs
 gatsby-*
 node_modules
 public
-scripts
 src

scripts/lib/file-helper.cjs

@@ -0,0 +1,157 @@
+const path = require('path');
+const fs = require('fs');
+const matcher = require('./matcher.cjs');
+
+function isFile(path, followSymlinks = true){
+    const stat = fs.lstatSync(path);
+    if(stat.isFile()) return true;
+    if(followSymlinks && stat.isSymbolicLink()){
+        return fs.lstatSync(fs.realpathSync(path)).isFile();
+    }
+    return false;
+}
+
+function isDirectory(path, followSymlinks = true){
+    const stat = fs.lstatSync(path);
+    if(stat.isDirectory()) return true;
+    if(followSymlinks && stat.isSymbolicLink()){
+        return fs.lstatSync(fs.realpathSync(path)).isDirectory();
+    }
+    return false;
+}
+
+/**
+ * Returns all subdirectories (non-recursive) of a given path
+ * which don't match an exclude pattern
+ * @param {String} startPath 
+ * @param {String[]} excludePatterns
+ * @param {boolean} followSymlinks Defines if symlinks should be considered
+ * @returns a string array containing the full paths of the subdirectories
+ */
+function getSubdirectories(startPath, excludePatterns = [], followSymlinks = true){
+    if (!fs.existsSync(startPath)) {        
+        console.log("💣 Directory doesn't exist:", startPath);
+        return;
+    }
+
+    var files = fs.readdirSync(startPath);
+    let directories = [];
+    files.forEach(file => {
+        var fullPath = path.join(startPath, file);
+
+        if (matcher.matchAny(fullPath, excludePatterns)) {            
+            return;
+        }
+
+        if (isDirectory(fullPath, followSymlinks)) {
+            directories.push(fullPath);
+        }
+    })
+    return directories;    
+}
+
+/**
+ * Returns all file names in a path (non-recursive) with a given file extension.
+ * This function only returns the file names without path info.
+ * @param {*} path 
+ * @param {*} extension 
+ * @returns a string array containing the file names
+ */
+function getFilesWithExtension(path, extension){
+    return fs.readdirSync(path).filter(aFile => aFile.endsWith('.' + extension));
+}
+
+
+/**
+ * Returns a list of file paths based on the start path
+ * @param {*} startPath 
+ * @param {*} searchPattern 
+ * @param {*} excludePatterns
+ * @param {boolean} followSymlinks Defines if symlinks should be considered
+ * @param {*} matchingFiles 
+ * @returns 
+ */
+function findAllFilesAndFolders(startPath, searchPattern = null, excludePatterns = [], followSymlinks = true, matchingFiles = []) {
+    if(matcher.matchAny(startPath, excludePatterns)){
+        // console.log("Excluding directory " + startPath);
+        return matchingFiles;
+    }
+
+    // console.log('Starting from dir ' + startPath + '/');
+
+    if (!fs.existsSync(startPath)) {
+        console.log("💣 Directory doesn't exist:", startPath, " search pattern: ", searchPattern);
+        return null;
+    }
+
+    var files = fs.readdirSync(startPath);
+    for (let file of files) {
+        var filePath = path.join(startPath, file);
+
+        if (!matcher.matchAny(filePath, excludePatterns)) {
+            if(!searchPattern) {
+                matchingFiles.push(filePath);
+                continue;
+            }
+            let patterns = Array.isArray(searchPattern) ? searchPattern : [searchPattern];            
+            patterns.forEach(pattern => {
+                if(filePath.indexOf(pattern) >= 0){
+                    // console.log('-- found: ', filename);
+                    matchingFiles.push(filePath);
+                }
+            });
+
+            if (isDirectory(filePath, followSymlinks)) {
+                findAllFiles(filePath, searchPattern, excludePatterns, followSymlinks, matchingFiles);
+            }
+        };
+
+    };
+
+    return matchingFiles;
+}
+
+function findAllFolders(startPath, searchPattern, excludePatterns = [], followSymlinks = true, matchingFiles = []) {
+    return findAllFilesAndFolders(startPath, searchPattern, excludePatterns, followSymlinks, matchingFiles)?.filter(file => {
+        if(!isDirectory(file, followSymlinks)) return false;
+        const lastPathComponent = file.substring(file.lastIndexOf('/') + 1);
+        return matcher.matchAny(lastPathComponent, [searchPattern]);
+    });
+}
+
+/**
+ * 
+ * @param {String} startPath The directory from which to start a recursive search
+ * @param {String} searchPattern The file name that should be looked for
+ * @param {String[]} excludePatterns An array of paths that should be excluded from the search
+ * @param {boolean} followSymlinks Defines if symlinks should be considered
+ * @param {String[]} matchingFiles The matching files as recursion parameter
+ */
+function findAllFiles(startPath, searchPattern, excludePatterns = [], followSymlinks = true, matchingFiles = []) {    
+    return findAllFilesAndFolders(startPath, searchPattern, excludePatterns, followSymlinks, matchingFiles)?.filter(file => {
+        return isFile(file, followSymlinks);
+    });
+};
+
+function createDirectoryIfNecessary(path){
+    if(!fs.existsSync(path)){
+        fs.mkdirSync(path, { recursive: true });
+    }
+}
+
+function getLineNumberFromIndex(index, haystack){
+    const tempString = haystack.substring(0, index);
+    const lineNumber = tempString.split('\n').length;
+    return lineNumber;
+}
+
+function getColumnFromIndex(index, haystack){
+    const tempString = haystack.substring(0, index);
+    const lines = tempString.split('\n');
+    lines.pop();
+    const indexOfLastLine = lines.length > 0 ? lines.join('\n').length + 1 : 0;
+    const column = (index - indexOfLastLine) + 1;
+    return column;
+}
+
+module.exports = { findAllFiles, findAllFolders, getFilesWithExtension, getSubdirectories, createDirectoryIfNecessary, getLineNumberFromIndex, getColumnFromIndex};

scripts/lib/matcher.cjs

@@ -0,0 +1,39 @@
+/**
+ * Checks if a text matches any of the supplied patterns.
+ * @param {*} text 
+ * @param {*} patterns 
+ * @param {*} callback if specified the callback will be called with the pattern that was matched
+ * @returns a boolean indicating whether a pattern was matched.
+ */
+function matchAny(text, patterns, callback = null){
+    let result = false;
+
+    for(let pattern of patterns){
+        if(text.indexOf(pattern) != -1){
+            if(callback) callback(pattern);            
+            result = true;
+        }
+    }
+    return result;
+}
+
+/**
+ * Checks if the text matches all of the supplied patterns.
+ * @param {*} text 
+ * @param {*} patterns 
+ * @param {*} callback if specified the callback will be called with the pattern that was matched
+ * @returns a boolean indicating if all patterns were matched.
+ */
+function matchAll(text, patterns, callback = null){
+    let result = true;
+
+    for(let pattern of patterns){
+        if(text.indexOf(pattern) == -1){   
+            if(callback) callback(pattern);         
+            result = false;
+        }
+    }
+    return result;
+}
+
+module.exports = { matchAll, matchAny};

scripts/resources/spell-check-ignore-list.txt

@@ -0,0 +1,11 @@
+wan
+sais
+tre
+inout
+als
+leaded
+specif
+extint
+dout
+synopsys
+trun

scripts/validation/config/config-datasheets.yml

@@ -0,0 +1,13 @@
+---
+openingHeadingLevel: 2
+excludePatterns: [ ".git", "/template", ".DS_Store"]
+
+searchPatterns: [/datasheet/, /datasheets/]
+validateSyntaxSpecifiers: true
+
+validationRuleFiles: [./rules/rules-spelling.yml, ./rules/rules-trademarks.yml]
+
+allowNestedLists: false
+metadataSchema: rules/datasheet-metadata-schema.json
+
+brokenLinkExcludePatterns: [^./, ^../, ^#, ^chrome://, localhost , ^assets/, ^images/]

scripts/validation/config/config-generic.yml

@@ -0,0 +1,22 @@
+---
+allowedSyntaxSpecifiers: [arduino, bash, markup, clike, c, cpp, css, css-extras, java, javascript, jsx, js-extras, coffeescript, diff, git, go, graphql, handlebars, json, less, makefile, markdown, objectivec, ocaml, python, reason, sass, scss, sql, stylus, tsx, typescript, wasm, yaml]
+
+baseURL: "https://docs.arduino.cc"
+
+# The maximum amount of characters that a heading / title can have
+headingMaxLength: 60
+
+# The minimum amount of words that a valid image caption needs to contain
+imageCaptionMinWords: 2
+
+# Don't check for broken URLs by default
+checkForBrokenLinks: false
+
+# Validate metadata from Frontmatter by default
+validateMetadata: true
+
+# Allows for verbose output such as non-broken link status
+verbose: true
+
+# Allows to debug the rules by outputting process messages
+debug: false

scripts/validation/config/config-tutorials.yml

@@ -0,0 +1,13 @@
+---
+openingHeadingLevel: 2
+excludePatterns: [ ".git", "/template", ".DS_Store"]
+
+searchPatterns: [/tutorials/]
+validateSyntaxSpecifiers: true
+
+validationRuleFiles: [./rules/rules-spelling.yml, ./rules/rules-trademarks.yml, ./rules/rules-tutorials.yml]
+
+allowNestedLists: false
+metadataSchema: rules/tutorial-metadata-schema.json
+
+brokenLinkExcludePatterns: [^./, ^../, ^#, ^chrome://, localhost , ^assets/, ^images/]

scripts/validation/content-lint.cmd

@@ -0,0 +1,16 @@
+@ECHO OFF
+
+node -v
+IF %ERRORLEVEL% NEQ 0 (
+    cls
+    ECHO Please install Node.js from here https://nodejs.org/en/download/
+    EXIT /B 
+) else (
+    cls
+
+    ::Only install the modules if npm list return an Error
+    npm list --depth=0 || npm install && cls && ECHO Modules installed
+
+    :: argument %* only used with "Current directory" option
+    node validate.js %*
+)

scripts/validation/content-lint.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+if ! command -v node &> /dev/null
+then
+    echo "Please install Node.js from here https://nodejs.org/en/download/"
+    exit
+fi
+
+npm list --depth=0 > /dev/null 2>&1
+
+if [ $? -ne 0 ]; then
+    echo "Installing node modules..."
+    npm install    
+fi
+
+node validate.js "$@"