Skip to content

Commit bcbb3b0

Browse files
authored
Restore content validation script (#12)
* Restore content validation script * Add missing spell check ignore list
1 parent fff49b9 commit bcbb3b0

38 files changed

+10013
-1
lines changed
Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
name: Validate Content
2+
3+
on:
4+
repository_dispatch:
5+
types: deploy
6+
push:
7+
branches:
8+
- main
9+
pull_request:
10+
types: [opened, synchronize, ready_for_review, reopened]
11+
12+
jobs:
13+
spell_check:
14+
name: Spell Check
15+
runs-on: ubuntu-latest
16+
steps:
17+
- uses: actions/checkout@v1
18+
- name: Run spell check
19+
run: |
20+
pip install codespell
21+
codespell -I "scripts/resources/spell-check-ignore-list.txt" --skip="*.svg,*.dxf" ./content/
22+
23+
lint:
24+
name: Content Linter
25+
runs-on: ubuntu-latest
26+
strategy:
27+
matrix:
28+
node_version: [14]
29+
30+
steps:
31+
- uses: actions/checkout@v1
32+
33+
- name: Run linter
34+
run: |
35+
cd scripts/validation
36+
./content-lint.sh -p '../../content/hardware/'

.gitignore

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,5 +6,4 @@ envs
66
gatsby-*
77
node_modules
88
public
9-
scripts
109
src

scripts/lib/file-helper.cjs

Lines changed: 157 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,157 @@
1+
const path = require('path');
2+
const fs = require('fs');
3+
const matcher = require('./matcher.cjs');
4+
5+
function isFile(path, followSymlinks = true){
6+
const stat = fs.lstatSync(path);
7+
if(stat.isFile()) return true;
8+
if(followSymlinks && stat.isSymbolicLink()){
9+
return fs.lstatSync(fs.realpathSync(path)).isFile();
10+
}
11+
return false;
12+
}
13+
14+
function isDirectory(path, followSymlinks = true){
15+
const stat = fs.lstatSync(path);
16+
if(stat.isDirectory()) return true;
17+
if(followSymlinks && stat.isSymbolicLink()){
18+
return fs.lstatSync(fs.realpathSync(path)).isDirectory();
19+
}
20+
return false;
21+
}
22+
23+
/**
24+
* Returns all subdirectories (non-recursive) of a given path
25+
* which don't match an exclude pattern
26+
* @param {String} startPath
27+
* @param {String[]} excludePatterns
28+
* @param {boolean} followSymlinks Defines if symlinks should be considered
29+
* @returns a string array containing the full paths of the subdirectories
30+
*/
31+
function getSubdirectories(startPath, excludePatterns = [], followSymlinks = true){
32+
if (!fs.existsSync(startPath)) {
33+
console.log("💣 Directory doesn't exist:", startPath);
34+
return;
35+
}
36+
37+
var files = fs.readdirSync(startPath);
38+
let directories = [];
39+
files.forEach(file => {
40+
var fullPath = path.join(startPath, file);
41+
42+
if (matcher.matchAny(fullPath, excludePatterns)) {
43+
return;
44+
}
45+
46+
if (isDirectory(fullPath, followSymlinks)) {
47+
directories.push(fullPath);
48+
}
49+
})
50+
return directories;
51+
}
52+
53+
/**
54+
* Returns all file names in a path (non-recursive) with a given file extension.
55+
* This function only returns the file names without path info.
56+
* @param {*} path
57+
* @param {*} extension
58+
* @returns a string array containing the file names
59+
*/
60+
function getFilesWithExtension(path, extension){
61+
return fs.readdirSync(path).filter(aFile => aFile.endsWith('.' + extension));
62+
}
63+
64+
65+
/**
66+
* Returns a list of file paths based on the start path
67+
* @param {*} startPath
68+
* @param {*} searchPattern
69+
* @param {*} excludePatterns
70+
* @param {boolean} followSymlinks Defines if symlinks should be considered
71+
* @param {*} matchingFiles
72+
* @returns
73+
*/
74+
function findAllFilesAndFolders(startPath, searchPattern = null, excludePatterns = [], followSymlinks = true, matchingFiles = []) {
75+
if(matcher.matchAny(startPath, excludePatterns)){
76+
// console.log("Excluding directory " + startPath);
77+
return matchingFiles;
78+
}
79+
80+
// console.log('Starting from dir ' + startPath + '/');
81+
82+
if (!fs.existsSync(startPath)) {
83+
console.log("💣 Directory doesn't exist:", startPath, " search pattern: ", searchPattern);
84+
return null;
85+
}
86+
87+
var files = fs.readdirSync(startPath);
88+
for (let file of files) {
89+
var filePath = path.join(startPath, file);
90+
91+
if (!matcher.matchAny(filePath, excludePatterns)) {
92+
if(!searchPattern) {
93+
matchingFiles.push(filePath);
94+
continue;
95+
}
96+
let patterns = Array.isArray(searchPattern) ? searchPattern : [searchPattern];
97+
patterns.forEach(pattern => {
98+
if(filePath.indexOf(pattern) >= 0){
99+
// console.log('-- found: ', filename);
100+
matchingFiles.push(filePath);
101+
}
102+
});
103+
104+
if (isDirectory(filePath, followSymlinks)) {
105+
findAllFiles(filePath, searchPattern, excludePatterns, followSymlinks, matchingFiles);
106+
}
107+
};
108+
109+
};
110+
111+
return matchingFiles;
112+
}
113+
114+
function findAllFolders(startPath, searchPattern, excludePatterns = [], followSymlinks = true, matchingFiles = []) {
115+
return findAllFilesAndFolders(startPath, searchPattern, excludePatterns, followSymlinks, matchingFiles)?.filter(file => {
116+
if(!isDirectory(file, followSymlinks)) return false;
117+
const lastPathComponent = file.substring(file.lastIndexOf('/') + 1);
118+
return matcher.matchAny(lastPathComponent, [searchPattern]);
119+
});
120+
}
121+
122+
/**
123+
*
124+
* @param {String} startPath The directory from which to start a recursive search
125+
* @param {String} searchPattern The file name that should be looked for
126+
* @param {String[]} excludePatterns An array of paths that should be excluded from the search
127+
* @param {boolean} followSymlinks Defines if symlinks should be considered
128+
* @param {String[]} matchingFiles The matching files as recursion parameter
129+
*/
130+
function findAllFiles(startPath, searchPattern, excludePatterns = [], followSymlinks = true, matchingFiles = []) {
131+
return findAllFilesAndFolders(startPath, searchPattern, excludePatterns, followSymlinks, matchingFiles)?.filter(file => {
132+
return isFile(file, followSymlinks);
133+
});
134+
};
135+
136+
function createDirectoryIfNecessary(path){
137+
if(!fs.existsSync(path)){
138+
fs.mkdirSync(path, { recursive: true });
139+
}
140+
}
141+
142+
function getLineNumberFromIndex(index, haystack){
143+
const tempString = haystack.substring(0, index);
144+
const lineNumber = tempString.split('\n').length;
145+
return lineNumber;
146+
}
147+
148+
function getColumnFromIndex(index, haystack){
149+
const tempString = haystack.substring(0, index);
150+
const lines = tempString.split('\n');
151+
lines.pop();
152+
const indexOfLastLine = lines.length > 0 ? lines.join('\n').length + 1 : 0;
153+
const column = (index - indexOfLastLine) + 1;
154+
return column;
155+
}
156+
157+
module.exports = { findAllFiles, findAllFolders, getFilesWithExtension, getSubdirectories, createDirectoryIfNecessary, getLineNumberFromIndex, getColumnFromIndex};

scripts/lib/matcher.cjs

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
/**
2+
* Checks if a text matches any of the supplied patterns.
3+
* @param {*} text
4+
* @param {*} patterns
5+
* @param {*} callback if specified the callback will be called with the pattern that was matched
6+
* @returns a boolean indicating whether a pattern was matched.
7+
*/
8+
function matchAny(text, patterns, callback = null){
9+
let result = false;
10+
11+
for(let pattern of patterns){
12+
if(text.indexOf(pattern) != -1){
13+
if(callback) callback(pattern);
14+
result = true;
15+
}
16+
}
17+
return result;
18+
}
19+
20+
/**
21+
* Checks if the text matches all of the supplied patterns.
22+
* @param {*} text
23+
* @param {*} patterns
24+
* @param {*} callback if specified the callback will be called with the pattern that was matched
25+
* @returns a boolean indicating if all patterns were matched.
26+
*/
27+
function matchAll(text, patterns, callback = null){
28+
let result = true;
29+
30+
for(let pattern of patterns){
31+
if(text.indexOf(pattern) == -1){
32+
if(callback) callback(pattern);
33+
result = false;
34+
}
35+
}
36+
return result;
37+
}
38+
39+
module.exports = { matchAll, matchAny};
Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
wan
2+
sais
3+
tre
4+
inout
5+
als
6+
leaded
7+
specif
8+
extint
9+
dout
10+
synopsys
11+
trun
Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
---
2+
openingHeadingLevel: 2
3+
excludePatterns: [ ".git", "/template", ".DS_Store"]
4+
5+
searchPatterns: [/datasheet/, /datasheets/]
6+
validateSyntaxSpecifiers: true
7+
8+
validationRuleFiles: [./rules/rules-spelling.yml, ./rules/rules-trademarks.yml]
9+
10+
allowNestedLists: false
11+
metadataSchema: rules/datasheet-metadata-schema.json
12+
13+
brokenLinkExcludePatterns: [^./, ^../, ^#, ^chrome://, localhost , ^assets/, ^images/]
Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
---
2+
allowedSyntaxSpecifiers: [arduino, bash, markup, clike, c, cpp, css, css-extras, java, javascript, jsx, js-extras, coffeescript, diff, git, go, graphql, handlebars, json, less, makefile, markdown, objectivec, ocaml, python, reason, sass, scss, sql, stylus, tsx, typescript, wasm, yaml]
3+
4+
baseURL: "https://docs.arduino.cc"
5+
6+
# The maximum amount of characters that a heading / title can have
7+
headingMaxLength: 60
8+
9+
# The minimum amount of words that a valid image caption needs to contain
10+
imageCaptionMinWords: 2
11+
12+
# Don't check for broken URLs by default
13+
checkForBrokenLinks: false
14+
15+
# Validate metadata from Frontmatter by default
16+
validateMetadata: true
17+
18+
# Allows for verbose output such as non-broken link status
19+
verbose: true
20+
21+
# Allows to debug the rules by outputting process messages
22+
debug: false
Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
---
2+
openingHeadingLevel: 2
3+
excludePatterns: [ ".git", "/template", ".DS_Store"]
4+
5+
searchPatterns: [/tutorials/]
6+
validateSyntaxSpecifiers: true
7+
8+
validationRuleFiles: [./rules/rules-spelling.yml, ./rules/rules-trademarks.yml, ./rules/rules-tutorials.yml]
9+
10+
allowNestedLists: false
11+
metadataSchema: rules/tutorial-metadata-schema.json
12+
13+
brokenLinkExcludePatterns: [^./, ^../, ^#, ^chrome://, localhost , ^assets/, ^images/]

scripts/validation/content-lint.cmd

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
@ECHO OFF
2+
3+
node -v
4+
IF %ERRORLEVEL% NEQ 0 (
5+
cls
6+
ECHO Please install Node.js from here https://nodejs.org/en/download/
7+
EXIT /B
8+
) else (
9+
cls
10+
11+
::Only install the modules if npm list return an Error
12+
npm list --depth=0 || npm install && cls && ECHO Modules installed
13+
14+
:: argument %* only used with "Current directory" option
15+
node validate.js %*
16+
)

scripts/validation/content-lint.sh

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
#!/bin/bash
2+
3+
if ! command -v node &> /dev/null
4+
then
5+
echo "Please install Node.js from here https://nodejs.org/en/download/"
6+
exit
7+
fi
8+
9+
npm list --depth=0 > /dev/null 2>&1
10+
11+
if [ $? -ne 0 ]; then
12+
echo "Installing node modules..."
13+
npm install
14+
fi
15+
16+
node validate.js "$@"

0 commit comments

Comments
 (0)