Merge pull request #51 from Shopify/validate_admin_gql

WIP Refactor tools to better detect/correct hallucinations within code blocks

Author: Arkham

.vscode/settings.json

@@ -1,4 +1,7 @@
 {
   "editor.defaultFormatter": "esbenp.prettier-vscode",
-  "editor.formatOnSave": true
+  "editor.formatOnSave": true,
+  "editor.codeActionsOnSave": {
+    "source.organizeImports": "explicit"
+  }
 }

package-lock.json

@@ -18,13 +18,15 @@
   "description": "A command line tool for setting up Shopify Dev MCP server",
   "dependencies": {
     "@modelcontextprotocol/sdk": "^1.6.1",
+    "graphql": "^16.11.0",
     "zod": "^3.24.2"
   },
   "devDependencies": {
     "@changesets/changelog-github": "^0.5.1",
     "@changesets/cli": "^2.29.4",
     "@jest/globals": "^29.7.0",
     "@modelcontextprotocol/inspector": "^0.5.1",
+    "@types/graphql": "^14.2.3",
     "@types/node": "^22.13.10",
     "@vitest/coverage-v8": "^3.0.9",
     "memfs": "^4.17.0",

package.json

@@ -2,11 +2,11 @@
 
 import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
 import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
-import { shopifyTools } from "./tools/index.js";
-import { shopifyPrompts } from "./prompts/index.js";
 import { readFileSync } from "fs";
-import { fileURLToPath } from "url";
 import { dirname, resolve } from "path";
+import { fileURLToPath } from "url";
+import { shopifyPrompts } from "./prompts/index.js";
+import { shopifyTools } from "./tools/index.js";
 
 // Get package.json version
 const __filename = fileURLToPath(import.meta.url);

src/index.ts

@@ -1,5 +1,5 @@
-import { z } from "zod";
 import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
+import { z } from "zod";
 
 export function shopifyPrompts(server: McpServer) {
   server.prompt(

src/prompts/index.ts

@@ -1,22 +1,24 @@
 import {
+  afterAll,
+  afterEach,
+  beforeEach,
   describe,
+  expect,
   it,
   test,
-  expect,
-  beforeEach,
   vi,
-  afterAll,
-  afterEach,
 } from "vitest";
 
 global.fetch = vi.fn();
 
-import { shopifyTools, searchShopifyDocs } from "./index.js";
 import {
   instrumentationData,
   isInstrumentationDisabled,
 } from "../instrumentation.js";
-import { searchShopifyAdminSchema } from "./shopify-admin-schema.js";
+import { ValidationResult } from "../types.js";
+import validateGraphQLOperation from "../validations/graphqlSchema.js";
+import { searchShopifyDocs, shopifyTools } from "./index.js";
+import { searchShopifyAdminSchema } from "./shopifyAdminSchema.js";
 
 const originalConsoleError = console.error;
 const originalConsoleWarn = console.warn;
@@ -78,10 +80,15 @@ vi.mock("../instrumentation.js", () => ({
 }));
 
 // Mock searchShopifyAdminSchema
-vi.mock("./shopify-admin-schema.js", () => ({
+vi.mock("./shopifyAdminSchema.js", () => ({
   searchShopifyAdminSchema: vi.fn(),
 }));
 
+// Mock validateGraphQLOperation
+vi.mock("../validations/graphqlSchema.js", () => ({
+  default: vi.fn(),
+}));
+
 vi.mock("../../package.json", () => ({
   default: { version: "1.0.0" },
 }));
@@ -609,3 +616,249 @@ describe("get_started tool behavior", () => {
     expect(result.content[0].text).toContain("Network failure");
   });
 });
+
+describe("validate_graphql tool", () => {
+  let mockServer: any;
+  let validateGraphQLOperationMock: any;
+
+  beforeEach(() => {
+    vi.clearAllMocks();
+    validateGraphQLOperationMock = vi.mocked(validateGraphQLOperation);
+
+    // Mock fetch for getting started APIs
+    const fetchMock = global.fetch as any;
+    fetchMock.mockResolvedValue({
+      ok: true,
+      status: 200,
+      text: async () => JSON.stringify(sampleGettingStartedApisResponse),
+    });
+
+    // Create a mock server that captures the registered tools
+    mockServer = {
+      tool: vi.fn((name, description, schema, handler) => {
+        if (name === "validate_graphql") {
+          mockServer.validateHandler = handler;
+        }
+      }),
+      validateHandler: null,
+    };
+
+    // Mock instrumentation
+    vi.mocked(instrumentationData).mockReturnValue({
+      packageVersion: "1.0.0",
+      timestamp: "2024-01-01T00:00:00.000Z",
+    });
+    vi.mocked(isInstrumentationDisabled).mockReturnValue(false);
+  });
+
+  test("validates multiple code snippets successfully", async () => {
+    // Setup mock responses
+    validateGraphQLOperationMock
+      .mockResolvedValueOnce({
+        result: ValidationResult.SUCCESS,
+        resultDetail:
+          "Successfully validated GraphQL query against Shopify Admin API schema.",
+      })
+      .mockResolvedValueOnce({
+        result: ValidationResult.FAILED,
+        resultDetail:
+          "No GraphQL operation found in the provided code snippet.",
+      });
+
+    // Register the tools
+    await shopifyTools(mockServer);
+
+    // Ensure the handler was registered
+    expect(mockServer.validateHandler).not.toBeNull();
+
+    const testCodeSnippets = ["query { products { id } }", "const x = 1;"];
+
+    // Call the handler
+    const result = await mockServer.validateHandler({
+      code: testCodeSnippets,
+      api: "admin",
+    });
+
+    // Verify validateGraphQLOperation was called correctly
+    expect(validateGraphQLOperationMock).toHaveBeenCalledTimes(2);
+    expect(validateGraphQLOperationMock).toHaveBeenNthCalledWith(
+      1,
+      testCodeSnippets[0],
+      "admin",
+    );
+    expect(validateGraphQLOperationMock).toHaveBeenNthCalledWith(
+      2,
+      testCodeSnippets[1],
+      "admin",
+    );
+
+    // Verify the response
+    expect(result.content[0].type).toBe("text");
+    const responseText = result.content[0].text;
+    expect(responseText).toContain("❌ INVALID");
+    expect(responseText).toContain("**Total Code Snippets:** 2");
+    expect(responseText).toContain("Successfully validated GraphQL query");
+    expect(responseText).toContain("No GraphQL operation found");
+  });
+
+  test("handles validation failures correctly", async () => {
+    // Setup mock responses with failures
+    validateGraphQLOperationMock
+      .mockResolvedValueOnce({
+        result: ValidationResult.FAILED,
+        resultDetail:
+          "GraphQL validation errors: Cannot query field 'invalidField' on type 'Product'.",
+      })
+      .mockResolvedValueOnce({
+        result: ValidationResult.SUCCESS,
+        resultDetail:
+          "Successfully validated GraphQL mutation against Shopify Admin API schema.",
+      });
+
+    // Register the tools
+    await shopifyTools(mockServer);
+
+    const testCodeSnippets = [
+      "query { products { invalidField } }",
+      "mutation { productCreate(input: {}) { product { id } } }",
+    ];
+
+    // Call the handler
+    const result = await mockServer.validateHandler({
+      code: testCodeSnippets,
+      api: "admin",
+    });
+
+    // Verify the response shows invalid overall status
+    const responseText = result.content[0].text;
+    expect(responseText).toContain("❌ INVALID");
+    expect(responseText).toContain("**Total Code Snippets:** 2");
+    expect(responseText).toContain("Cannot query field 'invalidField'");
+    expect(responseText).toContain("Successfully validated GraphQL mutation");
+  });
+
+  test("handles mixed validation results", async () => {
+    // Setup mixed results
+    validateGraphQLOperationMock
+      .mockResolvedValueOnce({
+        result: ValidationResult.SUCCESS,
+        resultDetail:
+          "Successfully validated GraphQL query against Shopify Admin API schema.",
+      })
+      .mockResolvedValueOnce({
+        result: ValidationResult.FAILED,
+        resultDetail:
+          "No GraphQL operation found in the provided code snippet.",
+      })
+      .mockResolvedValueOnce({
+        result: ValidationResult.FAILED,
+        resultDetail:
+          "GraphQL syntax error: Syntax Error: Expected Name, found }",
+      });
+
+    // Register the tools
+    await shopifyTools(mockServer);
+
+    const testCodeSnippets = [
+      "query { products { id } }",
+      "const x = 1;",
+      "query { products { } }",
+    ];
+
+    // Call the handler
+    const result = await mockServer.validateHandler({
+      code: testCodeSnippets,
+      api: "admin",
+    });
+
+    // Verify the response shows invalid overall status due to failure
+    const responseText = result.content[0].text;
+    expect(responseText).toContain("❌ INVALID");
+    expect(responseText).toContain("**Total Code Snippets:** 3");
+    expect(responseText).toContain("Code Snippet 1\n**Status:** ✅ SUCCESS");
+    expect(responseText).toContain("Code Snippet 2\n**Status:** ❌ FAILED");
+    expect(responseText).toContain("Code Snippet 3\n**Status:** ❌ FAILED");
+    expect(responseText).toContain("Syntax Error: Expected Name, found }");
+  });
+
+  test("handles empty code snippets array", async () => {
+    // Register the tools
+    await shopifyTools(mockServer);
+
+    // Call the handler with empty array
+    const result = await mockServer.validateHandler({
+      code: [],
+      api: "admin",
+    });
+
+    // Verify validateGraphQLOperation was not called
+    expect(validateGraphQLOperationMock).not.toHaveBeenCalled();
+
+    // Verify the response
+    const responseText = result.content[0].text;
+    expect(responseText).toContain("✅ VALID");
+    expect(responseText).toContain("**Total Code Snippets:** 0");
+  });
+
+  test("handles validation function errors", async () => {
+    // Setup mock to throw an error
+    validateGraphQLOperationMock.mockRejectedValueOnce(
+      new Error("Schema loading failed"),
+    );
+
+    // Register the tools
+    await shopifyTools(mockServer);
+
+    const testCodeSnippets = ["query { products { id } }"];
+
+    // Call the handler and expect it to handle the error gracefully
+    await expect(
+      mockServer.validateHandler({
+        code: testCodeSnippets,
+        api: "admin",
+      }),
+    ).rejects.toThrow("Schema loading failed");
+
+    // Verify validateGraphQLOperation was called
+    expect(validateGraphQLOperationMock).toHaveBeenCalledTimes(1);
+  });
+
+  test("records usage data correctly", async () => {
+    // Setup successful validation
+    validateGraphQLOperationMock.mockResolvedValueOnce({
+      result: ValidationResult.SUCCESS,
+      resultDetail:
+        "Successfully validated GraphQL query against Shopify Admin API schema.",
+    });
+
+    // Mock fetch for usage recording
+    const fetchMock = global.fetch as any;
+    fetchMock.mockResolvedValue({
+      ok: true,
+      status: 200,
+    });
+
+    // Register the tools
+    await shopifyTools(mockServer);
+
+    const testCodeSnippets = ["query { products { id } }"];
+
+    // Call the handler
+    await mockServer.validateHandler({
+      code: testCodeSnippets,
+      api: "admin",
+    });
+
+    // Verify usage was recorded (should be called for API list fetch and usage recording)
+    const usageCalls = fetchMock.mock.calls.filter((call: [string, any]) =>
+      call[0].includes("/mcp/usage"),
+    );
+    expect(usageCalls.length).toBe(1);
+
+    // Verify the usage data
+    const usageCall = usageCalls[0];
+    const usageBody = JSON.parse(usageCall[1].body);
+    expect(usageBody.tool).toBe("validate_graphql");
+    expect(usageBody.parameters).toBe("1 code snippets");
+  });
+});