feat: check duplicate files

Author: BCsabaEngine

src/consoleColor.ts

@@ -1,5 +1,5 @@
-export const greenLog = (s: string): string => `\u001B[32m ${s} \u001B[0m`;
+export const greenLog = (s: string): string => `\u001B[32m${s}\u001B[0m`;
 
-export const yellowLog = (s: string): string => `\u001B[33m ${s} \u001B[0m`;
+export const yellowLog = (s: string): string => `\u001B[33m${s}\u001B[0m`;
 
-export const redLog = (s: string): string => `\u001B[31m ${s} \u001B[0m`;
+export const redLog = (s: string): string => `\u001B[31m${s}\u001B[0m`;

src/file.ts

@@ -1,22 +1,43 @@
+import { createHash } from 'node:crypto';
+import { readFileSync } from 'node:fs';
 import path from 'node:path';
 
 import { globSync } from 'glob';
 
 import { cmdLine } from './commandLine';
-import { redLog } from './consoleColor';
+import { redLog, yellowLog } from './consoleColor';
 
-export const getFiles = (): string[] => {
-  let files = globSync('**/*', { cwd: cmdLine.sourcepath, nodir: true });
-  files = files.filter((filename) => {
+const findSimilarFiles = (files: Map<string, Buffer>): string[][] => {
+  const contentComparer: Map<string, string[]> = new Map();
+  for (const [filename, content] of files.entries()) {
+    const hash = createHash('md5').update(content.toString()).digest('hex');
+    if (contentComparer.has(hash)) contentComparer.get(hash)!.push(filename);
+    else contentComparer.set(hash, [filename]);
+  }
+
+  const result: string[][] = [];
+  for (const filenames of contentComparer.values()) if (filenames.length > 1) result.push(filenames);
+  return result;
+};
+
+export const getFiles = (): Map<string, Buffer> => {
+  let filenames = globSync('**/*', { cwd: cmdLine.sourcepath, nodir: true });
+  filenames = filenames.filter((filename) => {
     const extension = path.extname(filename);
     if (['.gz', '.brottli', '.br'].includes(extension)) {
       const original = filename.slice(0, -1 * extension.length);
-      if (files.includes(original)) {
+      if (filenames.includes(original)) {
         console.log(redLog(`${filename} skipped because is perhaps a compressed version of ${original}`));
         return false;
       }
     }
     return true;
   });
-  return files.sort();
+
+  const result: Map<string, Buffer> = new Map();
+  for (const filename of filenames)
+    result.set(filename, readFileSync(path.join(cmdLine.sourcepath, filename), { flag: 'r' }));
+  for (const sameFile of findSimilarFiles(result))
+    console.log(yellowLog(`${sameFile.join(', ')} files look like identical`));
+  return result;
 };

src/index.ts

@@ -1,6 +1,6 @@
 /* eslint-disable unicorn/prefer-string-replace-all */
 import { createHash } from 'node:crypto';
-import { mkdirSync, readFileSync, writeFileSync } from 'node:fs';
+import { mkdirSync, writeFileSync } from 'node:fs';
 import path from 'node:path';
 import { gzipSync } from 'node:zlib';
 
@@ -19,18 +19,22 @@ const summary = {
 
 const sources: CppCodeSources = [];
 const filesByExtension: ExtensionGroups = [];
+
+console.log('Collecting source files');
 const files = getFiles();
-if (files.length === 0) {
+if (files.size === 0) {
   console.error(`Directory ${cmdLine.sourcepath} is empty`);
   process.exit(1);
 }
 
-const rightPad = files.reduce((p, c) => (c.length > p ? c.length : p), 0);
-for (const file of files) {
-  const mime = lookup(file) || 'text/plain';
+console.log();
+console.log('Translation to header file');
+const longestFilename = [...files.keys()].reduce((p, c) => (c.length > p ? c.length : p), 0);
+for (const [originalFilename, content] of files) {
+  const mime = lookup(originalFilename) || 'text/plain';
   summary.filecount++;
 
-  const filename = file.replace(/\\/g, '/');
+  const filename = originalFilename.replace(/\\/g, '/');
   const dataname = filename.replace(/[./-]/g, '_');
   let extension = path.extname(filename).toUpperCase();
   if (extension.startsWith('.')) extension = extension.slice(1);
@@ -39,7 +43,6 @@ for (const file of files) {
   if (group) group.count += 1;
   else filesByExtension.push({ extension, count: 1 });
 
-  const content = readFileSync(path.join(cmdLine.sourcepath, file), { flag: 'r' });
   const md5 = createHash('md5').update(content).digest('hex');
   summary.size += content.length;
   const zipContent = gzipSync(content, { level: 9 });
@@ -58,7 +61,7 @@ for (const file of files) {
     });
     console.log(
       greenLog(
-        `[${file}] ${' '.repeat(rightPad - file.length)} ✓ gzip used (${content.length} -> ${zipContent.length} = ${zipRatio}%)`
+        `[${originalFilename}] ${' '.repeat(longestFilename - originalFilename.length)} ✓ gzip used (${content.length} -> ${zipContent.length} = ${zipRatio}%)`
       )
     );
   } else {
@@ -74,7 +77,7 @@ for (const file of files) {
     });
     console.log(
       yellowLog(
-        `[${file}] ${' '.repeat(rightPad - file.length)} x gzip unused ${content.length <= 1024 ? `(too small) ` : ''}(${content.length} -> ${zipContent.length} = ${zipRatio}%)`
+        `[${originalFilename}] ${' '.repeat(longestFilename - originalFilename.length)} x gzip unused ${content.length <= 1024 ? `(too small) ` : ''}(${content.length} -> ${zipContent.length} = ${zipRatio}%)`
       )
     );
   }