Support schema and doc errors in liquid tools

Author: aswamy

src/tools/index.ts

@@ -557,9 +557,15 @@ function liquidMcpTools(server: McpServer) {
     return;
   }
 
+  const themeRepositoryDescription = `A theme repository is a directory that MUST contain the following directories: snippets, sections, config, templates. It can optionally contain assets, locales, blocks, layouts.`;
+
   server.tool(
     "validate_theme_codeblocks",
-    `This tool validates Liquid codeblocks, Liquid files, and supporting Theme files (e.g. JSON translation files, JSON config files, JSON template files, JavaScript files, CSS files, and SVG files) generated or updated by LLMs to ensure they don't have hallucinated Liquid filters, or incorrect references. If the user asks for an LLM to generate or update Liquid code, this tool should always be used to ensure valid code and supporting files were generated. If the codeblocks reference other files, those files must also be validated by this tool.`,
+    `This tool validates Liquid codeblocks, Liquid files, and supporting Theme files (e.g. JSON locale files, JSON config files, JSON template files, JavaScript files, CSS files, and SVG files) generated or updated by LLMs to ensure they don't have hallucinated Liquid content, or incorrect references.
+
+    DO NOT use this tool if the user is asking to add, update, or delete files within a Theme repository - Use \`validate_theme\` instead. ${themeRepositoryDescription}
+
+    Provide every codeblock that was generated or updated by the LLM to this tool.`,
 
     withConversationId({
       codeblocks: z
@@ -620,7 +626,7 @@ function liquidMcpTools(server: McpServer) {
 
   server.tool(
     "validate_theme",
-    `This tool MUST run if the user asks the LLM to create or modify Liquid code inside their Theme repository. A theme repository is a directory that MUST contain the following directories: snippets, sections, config, templates. It can optionally contain assets, locales, blocks, layouts.
+    `This tool MUST run if the user asks the LLM to create or modify Liquid code inside their Theme repository. ${themeRepositoryDescription}
 
     Only fix the errors in the files that are directly related to the user's prompt. Offer to fix other errors if the user asks for it.`,
 

src/validations/theme.test.ts

@@ -7,16 +7,19 @@ import validateTheme from "./theme.js";
 
 describe("validateTheme", () => {
   let tempThemeDirectory: string;
+  let blocksDirectory: string;
   let snippetsDirectory: string;
   let localesDirectory: string;
 
   beforeEach(async () => {
     tempThemeDirectory = await mkdtemp(join(tmpdir(), "theme-test-"));
 
+    blocksDirectory = join(tempThemeDirectory, "blocks");
     snippetsDirectory = join(tempThemeDirectory, "snippets");
     localesDirectory = join(tempThemeDirectory, "locales");
 
     await Promise.all([
+      mkdir(blocksDirectory, { recursive: true }),
       mkdir(snippetsDirectory, { recursive: true }),
       mkdir(localesDirectory, { recursive: true }),
     ]);
@@ -41,7 +44,7 @@ describe("validateTheme", () => {
     );
   });
 
-  it("should fail to validate a theme", async () => {
+  it("should fail to validate a theme with an unknown filter", async () => {
     // Create the test.liquid file with the specified content
     const liquidFile = join(snippetsDirectory, "test.liquid");
     await writeFile(liquidFile, "{{ 'hello' | non-existent-filter }}");
@@ -56,6 +59,50 @@ describe("validateTheme", () => {
     );
   });
 
+  it("should fail to validate a theme with an invalid schema", async () => {
+    // Create the test.liquid file with the specified content
+    const liquidFile = join(blocksDirectory, "test.liquid");
+    const schemaName = "Long long long long long long name";
+    await writeFile(
+      liquidFile,
+      `
+{% schema %}
+  {
+    "name": "${schemaName}"
+  }
+{% endschema %}`,
+    );
+
+    // Run validateTheme on the temporary directory
+    const result = await validateTheme(tempThemeDirectory);
+
+    // Assert the response was a success
+    expect(result.result).toBe(ValidationResult.FAILED);
+    expect(result.resultDetail).toContain(
+      `Schema name '${schemaName}' is too long (max 25 characters)`,
+    );
+  });
+
+  it("should fail to validate a theme with LiquidDoc errors", async () => {
+    const snippetFile = join(snippetsDirectory, "example-snippet.liquid");
+    await writeFile(
+      snippetFile,
+      "{% doc %} @param {string} param {% enddoc %} {{ param }}",
+    );
+
+    const liquidFile = join(blocksDirectory, "test.liquid");
+    await writeFile(liquidFile, `{% render 'example-snippet' %}`);
+
+    // Run validateTheme on the temporary directory
+    const result = await validateTheme(tempThemeDirectory);
+
+    // Assert the response was a success
+    expect(result.result).toBe(ValidationResult.FAILED);
+    expect(result.resultDetail).toContain(
+      `Missing required argument 'param' in render tag for snippet 'example-snippet'.`,
+    );
+  });
+
   it("should successfully validate a theme with an unknown filter if its check is exempted", async () => {
     // Create the test.liquid file with the specified content
     const liquidFile = join(snippetsDirectory, "test.liquid");

src/validations/themeCodeBlock.test.ts

@@ -168,4 +168,58 @@ describe("validateThemeCodeblocks", () => {
         "Theme codeblock test.liquid has the following offenses from using Shopify's Theme Check:\n\nERROR: 'snippets/non-existent-snippet.liquid' does not exist",
     });
   });
+
+  it("should fail to validate liquid code with schema errors", async () => {
+    const schemaName = "Long long long long long long name";
+    const codeblocks = [
+      {
+        fileName: "test.liquid",
+        fileType: "blocks" as const,
+        content: `
+{% schema %}
+  {
+    "name": "${schemaName}"
+  }
+{% endschema %}`,
+      },
+    ];
+
+    const result = await validateThemeCodeblocks(codeblocks);
+
+    expect(result).toContainEqual({
+      result: ValidationResult.FAILED,
+      resultDetail: `Theme codeblock test.liquid has the following offenses from using Shopify's Theme Check:
+
+ERROR: Schema name '${schemaName}' is too long (max 25 characters)`,
+    });
+  });
+
+  it("should fail to validate liquid code with LiquidDoc errors", async () => {
+    const codeblocks = [
+      {
+        fileName: "example-snippet.liquid",
+        fileType: "snippets" as const,
+        content: `
+{% doc %}
+  @param {string} param
+{% enddoc %}
+{{ param }} 
+`,
+      },
+      {
+        fileName: "test.liquid",
+        fileType: "blocks" as const,
+        content: `{% render 'example-snippet' %}`,
+      },
+    ];
+
+    const result = await validateThemeCodeblocks(codeblocks);
+
+    expect(result).toContainEqual({
+      result: ValidationResult.FAILED,
+      resultDetail: `Theme codeblock test.liquid has the following offenses from using Shopify's Theme Check:
+
+ERROR: Missing required argument 'param' in render tag for snippet 'example-snippet'.; SUGGESTED FIXES: Add required argument 'param'`,
+    });
+  });
 });

src/validations/themeCodeBlock.ts

@@ -2,15 +2,22 @@ import {
   AbstractFileSystem,
   check,
   Config,
+  extractDocDefinition,
   FileStat,
   FileTuple,
   FileType,
+  LiquidHtmlNode,
   Offense,
   path,
   recommended,
+  SectionSchema,
+  SourceCodeType,
+  ThemeBlockSchema,
+  toSchema,
   toSourceCode,
 } from "@shopify/theme-check-common";
 import { ThemeLiquidDocsManager } from "@shopify/theme-check-docs-updater";
+import { normalize } from "path";
 import { ValidationResponse, ValidationResult } from "../types.js";
 
 type ThemeCodeblock = {
@@ -95,6 +102,50 @@ async function runThemeCheck(theme: Theme) {
     fs: mockFs,
     themeDocset: docsManager,
     jsonValidationSet: docsManager,
+    getBlockSchema: async (blockName) => {
+      const blockUri = `file:///blocks/${blockName}.liquid`;
+      const sourceCode = themeSourceCode.find((s) => s.uri === blockUri);
+
+      if (!sourceCode) {
+        return undefined;
+      }
+
+      return toSchema(
+        "theme",
+        blockUri,
+        sourceCode,
+        async () => true,
+      ) as Promise<ThemeBlockSchema | undefined>;
+    },
+    getSectionSchema: async (sectionName) => {
+      const sectionUri = `file:///sections/${sectionName}.liquid`;
+      const sourceCode = themeSourceCode.find((s) => s.uri === sectionUri);
+
+      if (!sourceCode) {
+        return undefined;
+      }
+
+      return toSchema(
+        "theme",
+        sectionUri,
+        sourceCode,
+        async () => true,
+      ) as Promise<SectionSchema | undefined>;
+    },
+    async getDocDefinition(relativePath) {
+      const sourceCode = themeSourceCode.find((s) =>
+        normalize(s.uri).endsWith(normalize(relativePath)),
+      );
+
+      if (!sourceCode || sourceCode.type !== SourceCodeType.LiquidHtml) {
+        return undefined;
+      }
+
+      return extractDocDefinition(
+        sourceCode.uri,
+        sourceCode.ast as LiquidHtmlNode,
+      );
+    },
   });
 }