Check for duplicate files enhancement (#24)

Author: BCsabaEngine

.coderabbit.yaml

@@ -0,0 +1,75 @@
+language: en-US
+tone_instructions: ''
+early_access: false
+enable_free_tier: true
+reviews:
+  profile: chill
+  request_changes_workflow: false
+  high_level_summary: true
+  high_level_summary_placeholder: '@coderabbitai summary'
+  auto_title_placeholder: '@coderabbitai'
+  review_status: true
+  poem: false
+  collapse_walkthrough: false
+  sequence_diagrams: true
+  path_filters: []
+  path_instructions: []
+  abort_on_close: true
+  auto_review:
+    enabled: true
+    auto_incremental_review: true
+    ignore_title_keywords: []
+    labels: []
+    drafts: false
+    base_branches: []
+  tools:
+    shellcheck:
+      enabled: true
+    ruff:
+      enabled: true
+    markdownlint:
+      enabled: true
+    github-checks:
+      enabled: true
+      timeout_ms: 90000
+    languagetool:
+      enabled: true
+      enabled_only: false
+      level: default
+    biome:
+      enabled: true
+    hadolint:
+      enabled: true
+    swiftlint:
+      enabled: true
+    phpstan:
+      enabled: true
+      level: default
+    golangci-lint:
+      enabled: true
+    yamllint:
+      enabled: true
+    gitleaks:
+      enabled: true
+    checkov:
+      enabled: true
+    detekt:
+      enabled: true
+    eslint:
+      enabled: true
+    rubocop:
+      enabled: true
+    buf:
+      enabled: true
+chat:
+  auto_reply: true
+knowledge_base:
+  opt_out: false
+  learnings:
+    scope: auto
+  issues:
+    scope: auto
+  jira:
+    project_keys: []
+  linear:
+    team_keys: []

demo/esp32/src/main.cpp

@@ -6,7 +6,7 @@
 #include <ESPAsyncWebServer.h>
 #include "svelteesp32async.h"
 
-#if SVELTEESP32_COUNT != 10
+#if SVELTEESP32_COUNT != 11
 #error Invalid file count
 #endif
 
@@ -40,7 +40,7 @@ void loop() {}
 #include <PsychicHttp.h>
 #include "svelteesp32psychic.h"
 
-#if SVELTEESP32_COUNT != 10
+#if SVELTEESP32_COUNT != 11
 #error Invalid file count
 #endif
 

demo/svelte/dist/favicon.png.gz

@@ -9,7 +9,7 @@
 	},
 	"scripts": {
 		"dev": "vite dev --host",
-		"build": "vite build",
+		"build": "vite build && gzip -c ./dist/favicon.png > ./dist/favicon.png.gz && cp ./dist/gallery/image-1.jpg ./dist/gallery/image-1a.jpg",
 		"preview": "vite preview",
 		"test": "npm run test:integration && npm run test:unit",
 		"ts:check": "svelte-check --tsconfig ./tsconfig.json",
@@ -23,8 +23,8 @@
 		"@rollup/plugin-swc": "^0.3.1",
 		"@sveltejs/vite-plugin-svelte": "^3.1.1",
 		"@tsconfig/svelte": "^5.0.4",
-		"@typescript-eslint/eslint-plugin": "^8.1.0",
-		"@typescript-eslint/parser": "^8.1.0",
+		"@typescript-eslint/eslint-plugin": "^8.2.0",
+		"@typescript-eslint/parser": "^8.2.0",
 		"autoprefixer": "^10.4.20",
 		"eslint": "^9.9.0",
 		"eslint-config-prettier": "^9.1.0",

demo/svelte/dist/gallery/image-1a.jpg

@@ -1,5 +1,5 @@
-export const greenLog = (s: string): string => `\u001B[32m ${s} \u001B[0m`;
+export const greenLog = (s: string): string => `\u001B[32m${s}\u001B[0m`;
 
-export const yellowLog = (s: string): string => `\u001B[33m ${s} \u001B[0m`;
+export const yellowLog = (s: string): string => `\u001B[33m${s}\u001B[0m`;
 
-export const redLog = (s: string): string => `\u001B[31m ${s} \u001B[0m`;
+export const redLog = (s: string): string => `\u001B[31m${s}\u001B[0m`;

demo/svelte/package-lock.json

@@ -1,22 +1,43 @@
+import { createHash } from 'node:crypto';
+import { readFileSync } from 'node:fs';
 import path from 'node:path';
 
 import { globSync } from 'glob';
 
 import { cmdLine } from './commandLine';
-import { redLog } from './consoleColor';
+import { redLog, yellowLog } from './consoleColor';
 
-export const getFiles = (): string[] => {
-  let files = globSync('**/*', { cwd: cmdLine.sourcepath, nodir: true });
-  files = files.filter((filename) => {
+const findSimilarFiles = (files: Map<string, Buffer>): string[][] => {
+  const contentComparer: Map<string, string[]> = new Map();
+  for (const [filename, content] of files.entries()) {
+    const hash = createHash('sha256').update(content).digest('hex');
+    if (contentComparer.has(hash)) contentComparer.get(hash)!.push(filename);
+    else contentComparer.set(hash, [filename]);
+  }
+
+  const result: string[][] = [];
+  for (const filenames of contentComparer.values()) if (filenames.length > 1) result.push(filenames);
+  return result;
+};
+
+export const getFiles = (): Map<string, Buffer> => {
+  let filenames = globSync('**/*', { cwd: cmdLine.sourcepath, nodir: true });
+  filenames = filenames.filter((filename) => {
     const extension = path.extname(filename);
     if (['.gz', '.brottli', '.br'].includes(extension)) {
       const original = filename.slice(0, -1 * extension.length);
-      if (files.includes(original)) {
-        console.log(redLog(`${filename} skipped because is perhaps a compressed version of ${original}`));
+      if (filenames.includes(original)) {
+        console.log(redLog(` ${filename} skipped because is perhaps a compressed version of ${original}`));
         return false;
       }
     }
     return true;
   });
-  return files.sort();
+
+  const result: Map<string, Buffer> = new Map();
+  for (const filename of filenames)
+    result.set(filename, readFileSync(path.join(cmdLine.sourcepath, filename), { flag: 'r' }));
+  for (const sameFile of findSimilarFiles(result))
+    console.log(yellowLog(` ${sameFile.join(', ')} files look like identical`));
+  return result;
 };