base logic

Author: andrewhong5297

package-lock.json

@@ -40,7 +40,8 @@
     "marked": "^4.3.0",
     "marked-terminal": "^5.1.1",
     "minimatch": "^7.4.3",
-    "ora": "^6.2.0"
+    "ora": "^6.2.0",
+    "ts-md5": "^1.3.1"
   },
   "devDependencies": {
     "@types/commander": "^2.12.2",

package.json

@@ -1,5 +1,6 @@
 import fs from 'node:fs/promises';
 import path from 'node:path';
+import { Md5 } from 'ts-md5';
 import { OpenAIChat } from 'langchain/llms';
 import { encoding_for_model } from '@dqbd/tiktoken';
 import { APIRateLimit } from '../../utils/APIRateLimit.js';
@@ -69,6 +70,19 @@ export const processRepository = async (
     linkHosted,
   }): Promise<void> => {
     const content = await fs.readFile(filePath, 'utf-8');
+
+    //calculate the hash of the file
+    const newChecksum = await calculateChecksum(filePath, [content]);
+
+    //if an existing summary.json file exists, it will check the checksums and decide if a reindex is needed
+    const reindex = await reindexCheck(
+      path.join(outputRoot, filePath),
+      newChecksum,
+    );
+    if (!reindex) {
+      return;
+    }
+
     const markdownFilePath = path.join(outputRoot, filePath);
     const url = githubFileUrl(repositoryUrl, inputRoot, filePath, linkHosted);
     const summaryPrompt = createCodeFileSummary(
@@ -140,6 +154,7 @@ export const processRepository = async (
           url,
           summary,
           questions,
+          checksum: newChecksum,
         };
 
         const outputPath = getFileName(markdownFilePath, '.', '.json');
@@ -195,6 +210,16 @@ export const processRepository = async (
     const contents = (await fs.readdir(folderPath)).filter(
       (fileName) => !shouldIgnore(fileName),
     );
+
+    //get the checksum of all the files in the folder
+    const newChecksum = await calculateChecksum(folderPath, contents);
+
+    //if an existing summary.json file exists, it will check the checksums and decide if a reindex is needed
+    const reindex = await reindexCheck(folderPath, newChecksum);
+    if (!reindex) {
+      return;
+    }
+
     // eslint-disable-next-line prettier/prettier
     const url = githubFolderUrl(repositoryUrl, inputRoot, folderPath, linkHosted);
     const allFiles: (FileSummary | null)[] = await Promise.all(
@@ -259,6 +284,7 @@ export const processRepository = async (
         folders: folders.filter(Boolean),
         summary,
         questions: '',
+        checksum: newChecksum,
       };
 
       const outputPath = path.join(folderPath, 'summary.json');
@@ -366,3 +392,51 @@ export const processRepository = async (
    */
   return models;
 };
+
+//reads all the files, and returns a checksum
+async function calculateChecksum(
+  folderPath: string,
+  contents: string[],
+): Promise<string> {
+  const checksums: string[] = [];
+  for (const fileName of contents) {
+    const filePath = `${folderPath}/${fileName}`;
+    const fileData = await fs.readFile(filePath, 'utf-8');
+    const checksum = Md5.hashStr(fileData);
+    checksums.push(checksum);
+  }
+  const concatenatedChecksum = checksums.join('');
+  const finalChecksum = Md5.hashStr(concatenatedChecksum);
+  return finalChecksum;
+}
+
+//checks if a summary.json file exists, and if it does, compares the checksums to see if it needs to be re-indexed or not.
+async function reindexCheck(
+  fileOrFolderPath: string,
+  newChecksum: string,
+): Promise<boolean> {
+  let summaryExists = false;
+  try {
+    await fs.access(path.join(fileOrFolderPath, 'summary.json'));
+    summaryExists = true;
+  } catch (error) {}
+
+  if (summaryExists) {
+    const fileContents = await fs.readFile(
+      path.join(fileOrFolderPath, 'summary.json'),
+      'utf8',
+    );
+    const fileContentsJSON = JSON.parse(fileContents);
+
+    const oldChecksum = fileContentsJSON.checksum;
+
+    if (oldChecksum === newChecksum) {
+      console.log(`Skipping ${fileOrFolderPath} because it has not changed`);
+      return false;
+    } else {
+      return true;
+    }
+  }
+  //if no summary then generate one
+  return true;
+}

src/cli/commands/index/processRepository.ts

@@ -36,6 +36,7 @@ export const traverseFileSystem = async (
             await dfs(folderPath);
 
             await processFolder?.({
+              inputPath,
               folderName,
               folderPath,
               projectName,

src/cli/utils/traverseFileSystem.ts

@@ -25,6 +25,7 @@ export type FileSummary = {
   url: string;
   summary: string;
   questions: string;
+  checksum: string;
 };
 
 export type ProcessFileParams = {
@@ -47,9 +48,11 @@ export type FolderSummary = {
   folders: FolderSummary[];
   summary: string;
   questions: string;
+  checksum: string;
 };
 
 export type ProcessFolderParams = {
+  inputPath: string;
   folderName: string;
   folderPath: string;
   projectName: string;