Merge pull request #10 from andrewhong5297/prompt-config

Introduce Prompt customization into config

Author: samheutmaker

.eslintrc.cjs

@@ -17,6 +17,12 @@ module.exports = {
     'react/react-in-jsx-scope': 'off',
     'react/prop-types': 'off',
     '@typescript-eslint/no-explicit-any': 'off',
+    'prettier/prettier': [
+      'error',
+      {
+        endOfLine: 'auto',
+      },
+    ],
   },
   // "overrides": [
   //   {

README.md

@@ -131,6 +131,8 @@ You will be prompted to enter the name of your project, GitHub url, and select w
 
 **Note:** Do not skip entering these values or indexing may not work.
 
+**Prompt Configuration:** You'll find prompt directions specified in `prompts.ts`, with some snippets customizable in the `autodoc.config.json`. The current prompts are developer focused and assume your repo is code focused. We will have more reference templates in the future.
+
 Run the `index` command:
 ```bash
 doc index

autodoc.config.json

@@ -20,5 +20,11 @@
     "*.mdx",
     "*.toml",
     "*autodoc*"
-  ]
+  ],
+  "filePrompt": "Write a detailed technical explanation of what this code does. \n      Focus on the high-level purpose of the code and how it may be used in the larger project.\n      Include code examples where appropriate. Keep you response between 100 and 300 words. \n      DO NOT RETURN MORE THAN 300 WORDS.\n      Output should be in markdown format.\n      Do not just list the methods and classes in this file.",
+  "folderPrompt": "Write a technical explanation of what the code in this file does\n      and how it might fit into the larger project or work with other parts of the project.\n      Give examples of how this code might be used. Include code examples where appropriate.\n      Be concise. Include any information that may be relevant to a developer who is curious about this code.\n      Keep you response under 400 words. Output should be in markdown format.\n      Do not just list the files and folders in this folder.",
+  "chatPrompt": "",
+  "contentType": "code",
+  "targetAudience": "smart developer",
+  "linkHosted": false
 }

package-lock.json

@@ -28,6 +28,7 @@
   "license": "MIT",
   "dependencies": {
     "@dqbd/tiktoken": "^1.0.2",
+    "@types/istextorbinary": "^2.3.1",
     "chalk": "^5.2.0",
     "cli-progress": "^3.12.0",
     "commander": "^10.0.0",

package.json

@@ -15,6 +15,12 @@ export const estimate = async ({
   output,
   llms,
   ignore,
+  filePrompt,
+  folderPrompt,
+  chatPrompt,
+  contentType,
+  targetAudience,
+  linkHosted,
 }: AutodocRepoConfig) => {
   const json = path.join(output, 'docs', 'json/');
 
@@ -32,6 +38,12 @@ export const estimate = async ({
       output: json,
       llms,
       ignore,
+      filePrompt,
+      folderPrompt,
+      chatPrompt,
+      contentType,
+      targetAudience,
+      linkHosted,
     },
     true,
   );

src/cli/commands/estimate/index.ts

@@ -14,6 +14,11 @@ export const convertJsonToMarkdown = async ({
   name: projectName,
   root: inputRoot,
   output: outputRoot,
+  filePrompt: filePrompt,
+  folderPrompt: folderPrompt,
+  contentType: contentType,
+  targetAudience: targetAudience,
+  linkHosted: linkHosted,
 }: AutodocRepoConfig) => {
   /**
    * Count the number of files in the project
@@ -27,6 +32,11 @@ export const convertJsonToMarkdown = async ({
       return Promise.resolve();
     },
     ignore: [],
+    filePrompt,
+    folderPrompt,
+    contentType,
+    targetAudience,
+    linkHosted,
   });
 
   /**
@@ -83,6 +93,11 @@ export const convertJsonToMarkdown = async ({
     projectName,
     processFile,
     ignore: [],
+    filePrompt,
+    folderPrompt,
+    contentType,
+    targetAudience,
+    linkHosted,
   });
   spinnerSuccess(`Created ${files} mardown files...`);
 };

src/cli/commands/index/convertJsonToMarkdown.ts

@@ -12,6 +12,12 @@ export const index = async ({
   output,
   llms,
   ignore,
+  filePrompt,
+  folderPrompt,
+  chatPrompt,
+  contentType,
+  targetAudience,
+  linkHosted,
 }: AutodocRepoConfig) => {
   const json = path.join(output, 'docs', 'json/');
   const markdown = path.join(output, 'docs', 'markdown/');
@@ -30,6 +36,12 @@ export const index = async ({
     output: json,
     llms,
     ignore,
+    filePrompt,
+    folderPrompt,
+    chatPrompt,
+    contentType,
+    targetAudience,
+    linkHosted,
   });
   updateSpinnerText('Processing repository...');
   spinnerSuccess();
@@ -45,6 +57,12 @@ export const index = async ({
     output: markdown,
     llms,
     ignore,
+    filePrompt,
+    folderPrompt,
+    chatPrompt,
+    contentType,
+    targetAudience,
+    linkHosted,
   });
   spinnerSuccess();
 
@@ -56,6 +74,12 @@ export const index = async ({
     output: data,
     llms,
     ignore,
+    filePrompt,
+    folderPrompt,
+    chatPrompt,
+    contentType,
+    targetAudience,
+    linkHosted,
   });
   spinnerSuccess();
 };

src/cli/commands/index/index.ts

@@ -38,6 +38,11 @@ export const processRepository = async (
     output: outputRoot,
     llms,
     ignore,
+    filePrompt,
+    folderPrompt,
+    contentType,
+    targetAudience,
+    linkHosted,
   }: AutodocRepoConfig,
   dryRun?: boolean,
 ) => {
@@ -58,19 +63,27 @@ export const processRepository = async (
     fileName,
     filePath,
     projectName,
+    contentType,
+    filePrompt,
+    targetAudience,
+    linkHosted,
   }): Promise<void> => {
     const content = await fs.readFile(filePath, 'utf-8');
     const markdownFilePath = path.join(outputRoot, filePath);
-    const url = githubFileUrl(repositoryUrl, inputRoot, filePath);
+    const url = githubFileUrl(repositoryUrl, inputRoot, filePath, linkHosted);
     const summaryPrompt = createCodeFileSummary(
       projectName,
       projectName,
       content,
+      contentType,
+      filePrompt,
     );
     const questionsPrompt = createCodeQuestions(
       projectName,
       projectName,
       content,
+      contentType,
+      targetAudience,
     );
     const summaryLength = encoding.encode(summaryPrompt).length;
     const questionLength = encoding.encode(questionsPrompt).length;
@@ -167,7 +180,10 @@ export const processRepository = async (
     folderName,
     folderPath,
     projectName,
+    contentType,
+    folderPrompt,
     shouldIgnore,
+    linkHosted,
   }): Promise<void> => {
     /**
      * For now we don't care about folders
@@ -179,7 +195,8 @@ export const processRepository = async (
     const contents = (await fs.readdir(folderPath)).filter(
       (fileName) => !shouldIgnore(fileName),
     );
-    const url = githubFolderUrl(repositoryUrl, inputRoot, folderPath);
+    // eslint-disable-next-line prettier/prettier
+    const url = githubFolderUrl(repositoryUrl, inputRoot, folderPath, linkHosted);
     const allFiles: (FileSummary | null)[] = await Promise.all(
       contents.map(async (fileName) => {
         const entryPath = path.join(folderPath, fileName);
@@ -223,7 +240,14 @@ export const processRepository = async (
       );
 
       const summary = await callLLM(
-        folderSummaryPrompt(folderPath, projectName, files, folders),
+        folderSummaryPrompt(
+          folderPath,
+          projectName,
+          files,
+          folders,
+          contentType,
+          folderPrompt,
+        ),
         models[LLMModels.GPT4].llm,
       );
 
@@ -252,7 +276,7 @@ export const processRepository = async (
   };
 
   /**
-   * Get the numver of files and folderfs in the project
+   * Get the number of files and folders in the project
    */
 
   const filesAndFolders = async (): Promise<{
@@ -271,6 +295,11 @@ export const processRepository = async (
           return Promise.resolve();
         },
         ignore,
+        filePrompt,
+        folderPrompt,
+        contentType,
+        targetAudience,
+        linkHosted,
       }),
       traverseFileSystem({
         inputPath: inputRoot,
@@ -280,6 +309,11 @@ export const processRepository = async (
           return Promise.resolve();
         },
         ignore,
+        filePrompt,
+        folderPrompt,
+        contentType,
+        targetAudience,
+        linkHosted,
       }),
     ]);
 
@@ -301,6 +335,11 @@ export const processRepository = async (
     projectName,
     processFile,
     ignore,
+    filePrompt,
+    folderPrompt,
+    contentType,
+    targetAudience,
+    linkHosted,
   });
   spinnerSuccess(`Processing ${files} files...`);
 
@@ -313,6 +352,11 @@ export const processRepository = async (
     projectName,
     processFolder,
     ignore,
+    filePrompt,
+    folderPrompt,
+    contentType,
+    targetAudience,
+    linkHosted,
   });
   spinnerSuccess(`Processing ${folders} folders... `);
   stopSpinner();