improvement(mothership): agent model dropdown validations, markers for recommended models (#4213)

* improvement(mothership): agent model dropdown validations, recommendation system

* mark a few more models:

* remove regex based checks'

* remove dead code

* remove inherited reseller flags

* fix note

* address bugbot comments

* code cleanup

Author: icecrasher321

apps/sim/lib/copilot/tools/server/workflow/edit-workflow/validation.test.ts

@@ -4,8 +4,9 @@
 import { beforeEach, describe, expect, it, vi } from 'vitest'
 import { normalizeConditionRouterIds } from './builders'
 
-const { mockValidateSelectorIds } = vi.hoisted(() => ({
+const { mockValidateSelectorIds, mockGetModelOptions } = vi.hoisted(() => ({
   mockValidateSelectorIds: vi.fn(),
+  mockGetModelOptions: vi.fn(() => []),
 }))
 
 const conditionBlockConfig = {
@@ -26,7 +27,24 @@ const routerBlockConfig = {
   type: 'router_v2',
   name: 'Router',
   outputs: {},
-  subBlocks: [{ id: 'routes', type: 'router-input' }],
+  subBlocks: [
+    { id: 'routes', type: 'router-input' },
+    { id: 'model', type: 'combobox', options: mockGetModelOptions },
+  ],
+}
+
+const agentBlockConfig = {
+  type: 'agent',
+  name: 'Agent',
+  outputs: {},
+  subBlocks: [{ id: 'model', type: 'combobox', options: mockGetModelOptions }],
+}
+
+const huggingfaceBlockConfig = {
+  type: 'huggingface',
+  name: 'HuggingFace',
+  outputs: {},
+  subBlocks: [{ id: 'model', type: 'short-input' }],
 }
 
 vi.mock('@/blocks/registry', () => ({
@@ -37,7 +55,15 @@ vi.mock('@/blocks/registry', () => ({
         ? oauthBlockConfig
         : type === 'router_v2'
           ? routerBlockConfig
-          : undefined,
+          : type === 'agent'
+            ? agentBlockConfig
+            : type === 'huggingface'
+              ? huggingfaceBlockConfig
+              : undefined,
+}))
+
+vi.mock('@/blocks/utils', () => ({
+  getModelOptions: mockGetModelOptions,
 }))
 
 vi.mock('@/lib/copilot/validation/selector-validator', () => ({
@@ -83,6 +109,105 @@ describe('validateInputsForBlock', () => {
     expect(result.errors).toHaveLength(1)
     expect(result.errors[0]?.error).toContain('expected a JSON array')
   })
+
+  it('accepts known agent model ids', () => {
+    const result = validateInputsForBlock('agent', { model: 'claude-sonnet-4-6' }, 'agent-1')
+
+    expect(result.errors).toHaveLength(0)
+    expect(result.validInputs.model).toBe('claude-sonnet-4-6')
+  })
+
+  it('rejects hallucinated agent model ids that match a static provider pattern', () => {
+    const result = validateInputsForBlock('agent', { model: 'claude-sonnet-4.6' }, 'agent-1')
+
+    expect(result.validInputs.model).toBeUndefined()
+    expect(result.errors).toHaveLength(1)
+    expect(result.errors[0]?.field).toBe('model')
+    expect(result.errors[0]?.error).toContain('Unknown model id')
+    expect(result.errors[0]?.error).toContain('claude-sonnet-4-6')
+  })
+
+  it('rejects legacy claude-4.5-haiku style ids', () => {
+    const result = validateInputsForBlock('agent', { model: 'claude-4.5-haiku' }, 'agent-1')
+
+    expect(result.errors).toHaveLength(1)
+    expect(result.errors[0]?.error).toContain('Unknown model id')
+  })
+
+  it('allows empty model values', () => {
+    const result = validateInputsForBlock('agent', { model: '' }, 'agent-1')
+
+    expect(result.errors).toHaveLength(0)
+    expect(result.validInputs.model).toBe('')
+  })
+
+  it('allows custom ollama-prefixed model ids', () => {
+    const result = validateInputsForBlock('agent', { model: 'ollama/my-private-model' }, 'agent-1')
+
+    expect(result.errors).toHaveLength(0)
+    expect(result.validInputs.model).toBe('ollama/my-private-model')
+  })
+
+  it('validates the model field on router_v2 blocks too', () => {
+    const valid = validateInputsForBlock('router_v2', { model: 'claude-sonnet-4-6' }, 'router-1')
+    expect(valid.errors).toHaveLength(0)
+    expect(valid.validInputs.model).toBe('claude-sonnet-4-6')
+
+    const invalid = validateInputsForBlock('router_v2', { model: 'claude-sonnet-4.6' }, 'router-1')
+    expect(invalid.validInputs.model).toBeUndefined()
+    expect(invalid.errors).toHaveLength(1)
+    expect(invalid.errors[0]?.blockType).toBe('router_v2')
+    expect(invalid.errors[0]?.field).toBe('model')
+    expect(invalid.errors[0]?.error).toContain('Unknown model id')
+  })
+
+  it("does not apply model validation to blocks whose model field is not Sim's catalog", () => {
+    const result = validateInputsForBlock(
+      'huggingface',
+      { model: 'mistralai/Mistral-7B-Instruct-v0.3' },
+      'hf-1'
+    )
+
+    expect(result.errors).toHaveLength(0)
+    expect(result.validInputs.model).toBe('mistralai/Mistral-7B-Instruct-v0.3')
+  })
+
+  it('rejects a bare Ollama-style tag without the provider prefix', () => {
+    const result = validateInputsForBlock('agent', { model: 'llama3.1:8b' }, 'agent-1')
+
+    expect(result.validInputs.model).toBeUndefined()
+    expect(result.errors).toHaveLength(1)
+    expect(result.errors[0]?.error).toContain('Unknown model id')
+    expect(result.errors[0]?.error).toContain('ollama/')
+  })
+
+  it('rejects date-pinned ids that are not literally in the catalog', () => {
+    const result = validateInputsForBlock(
+      'agent',
+      { model: 'claude-sonnet-4-5-20250929' },
+      'agent-1'
+    )
+
+    expect(result.validInputs.model).toBeUndefined()
+    expect(result.errors).toHaveLength(1)
+    expect(result.errors[0]?.error).toContain('Unknown model id')
+  })
+
+  it('trims whitespace around catalog model ids and stores the trimmed value', () => {
+    const result = validateInputsForBlock('agent', { model: '  gpt-5.4  ' }, 'agent-1')
+
+    expect(result.errors).toHaveLength(0)
+    expect(result.validInputs.model).toBe('gpt-5.4')
+  })
+
+  it('rejects a pattern-matching but uncataloged id even with surrounding whitespace', () => {
+    const result = validateInputsForBlock('agent', { model: '  gpt-100-ultra  ' }, 'agent-1')
+
+    expect(result.validInputs.model).toBeUndefined()
+    expect(result.errors).toHaveLength(1)
+    expect(result.errors[0]?.error).toContain('gpt-100-ultra')
+    expect(result.errors[0]?.error).not.toMatch(/\s{2,}/)
+  })
 })
 
 describe('normalizeConditionRouterIds', () => {

apps/sim/lib/copilot/tools/server/workflow/edit-workflow/validation.ts

@@ -3,7 +3,9 @@ import { validateSelectorIds } from '@/lib/copilot/validation/selector-validator
 import type { PermissionGroupConfig } from '@/lib/permission-groups/types'
 import { getBlock } from '@/blocks/registry'
 import type { SubBlockConfig } from '@/blocks/types'
+import { getModelOptions } from '@/blocks/utils'
 import { EDGE, normalizeName } from '@/executor/constants'
+import { isKnownModelId, suggestModelIdsForUnknownModel } from '@/providers/models'
 import { TRIGGER_RUNTIME_SUBBLOCK_IDS } from '@/triggers/constants'
 import type {
   EdgeHandleValidationResult,
@@ -350,9 +352,31 @@ export function validateValueForSubBlockType(
     case 'short-input':
     case 'long-input':
     case 'combobox': {
-      // Should be string (combobox allows custom values)
+      const usesProviderCatalog =
+        fieldName === 'model' && subBlockConfig.options === getModelOptions
+
+      if (usesProviderCatalog) {
+        const stringValue = typeof value === 'string' ? value : String(value)
+        const trimmed = stringValue.trim()
+        if (trimmed !== '' && !isKnownModelId(trimmed)) {
+          const suggestions = suggestModelIdsForUnknownModel(trimmed)
+          const suggestionText =
+            suggestions.length > 0 ? ` Valid options include: ${suggestions.join(', ')}.` : ''
+          return {
+            valid: false,
+            error: {
+              blockId,
+              blockType,
+              field: fieldName,
+              value,
+              error: `Unknown model id "${trimmed}" for block "${blockType}". Read components/blocks/${blockType}.json (the model.options array) for valid ids; prefer entries with recommended: true and avoid deprecated: true. For user-configured models (Ollama, vLLM, OpenRouter, Fireworks), prefix the id with the provider slash, e.g. "ollama/llama3.1:8b".${suggestionText}`,
+            },
+          }
+        }
+        return { valid: true, value: trimmed }
+      }
+
       if (typeof value !== 'string' && typeof value !== 'number') {
-        // Convert to string but don't error
         return { valid: true, value: String(value) }
       }
       return { valid: true, value }

apps/sim/lib/copilot/vfs/serializers.ts

@@ -2,7 +2,7 @@ import { getCopilotToolDescription } from '@/lib/copilot/tools/descriptions'
 import { isHosted } from '@/lib/core/config/feature-flags'
 import { isSubBlockHidden } from '@/lib/workflows/subblocks/visibility'
 import type { BlockConfig, SubBlockConfig } from '@/blocks/types'
-import { PROVIDER_DEFINITIONS } from '@/providers/models'
+import { DYNAMIC_MODEL_PROVIDERS, PROVIDER_DEFINITIONS } from '@/providers/models'
 import type { ToolConfig } from '@/tools/types'
 
 /**
@@ -320,24 +320,38 @@ export function serializeTableMeta(table: {
  * Excludes dynamic providers (ollama, vllm, openrouter) whose models are user-configured.
  * Includes provider ID and whether the model is hosted by Sim (no API key required).
  */
-function getStaticModelOptionsForVFS(): Array<{
+interface StaticModelOption {
   id: string
   provider: string
   hosted: boolean
-}> {
+  recommended?: boolean
+  speedOptimized?: boolean
+  deprecated?: boolean
+}
+
+const DYNAMIC_PROVIDERS_NOTE = {
+  note: 'The options array above lists Sim\'s static provider catalog. These providers also accept user-configured models that are NOT enumerated here: the user may have additional ids available at runtime (e.g. local Ollama tags). To reference one, prefix the model id with the provider slash below — for example "ollama/llama3.1:8b" instead of the bare "llama3.1:8b". The server rejects bare ids that are not in the catalog; always use the prefix for user-configured models.',
+  prefixes: DYNAMIC_MODEL_PROVIDERS.map((p) => `${p}/`),
+} as const
+
+function getStaticModelOptionsForVFS(): StaticModelOption[] {
   const hostedProviders = new Set(['openai', 'anthropic', 'google'])
-  const dynamicProviders = new Set(['ollama', 'vllm', 'openrouter', 'fireworks'])
+  const dynamicProviders = new Set<string>(DYNAMIC_MODEL_PROVIDERS)
 
-  const models: Array<{ id: string; provider: string; hosted: boolean }> = []
+  const models: StaticModelOption[] = []
 
   for (const [providerId, def] of Object.entries(PROVIDER_DEFINITIONS)) {
     if (dynamicProviders.has(providerId)) continue
     for (const model of def.models) {
-      models.push({
+      const option: StaticModelOption = {
         id: model.id,
         provider: providerId,
         hosted: hostedProviders.has(providerId),
-      })
+      }
+      if (model.recommended) option.recommended = true
+      if (model.speedOptimized) option.speedOptimized = true
+      if (model.deprecated) option.deprecated = true
+      models.push(option)
     }
   }
 
@@ -378,9 +392,9 @@ export function serializeBlockSchema(block: BlockConfig): string {
     .map((sb) => {
       const serialized = serializeSubBlock(sb)
 
-      // For model comboboxes with function options, inject static model data with hosting info
       if (sb.id === 'model' && sb.type === 'combobox' && typeof sb.options === 'function') {
         serialized.options = getStaticModelOptionsForVFS()
+        serialized.dynamicProviders = DYNAMIC_PROVIDERS_NOTE
       }
 
       return serialized