spb/tools-spboucher-ai

1{2  "slug": "promptfoo",3  "name": "promptfoo",4  "aliases": ["Promptfoo"],5  "vendor": { "name": "Promptfoo, Inc.", "slug": "promptfoo-inc", "website": "https://www.promptfoo.dev" },6  "homepage_url": "https://www.promptfoo.dev/",7  "docs_url": "https://www.promptfoo.dev/docs/intro/",8  "repo_url": "https://github.com/promptfoo/promptfoo",9  "pricing_url": "https://www.promptfoo.dev/pricing/",10  "license_url": "https://github.com/promptfoo/promptfoo/blob/main/LICENSE",11  "product_type": "cli-tool",12  "product_status": "active",13  "summary": "Open-source, local-first CLI and library for evaluating prompts, models, RAG pipelines and agents, with automated red teaming and CI integration",14  "description": "promptfoo is an MIT-licensed command-line tool and Node library for test-driven development of LLM applications. A configuration file declares prompts, providers and test cases with assertions; the tool runs the matrix, scores outputs with deterministic checks or model-graded metrics, caches results and renders side-by-side comparisons in a local web viewer. It speaks to OpenAI, Anthropic, Azure, Google, Hugging Face, local runtimes and custom HTTP providers, and it runs in CI pipelines so prompt changes are gated like code. A second major use is red teaming: automated adversarial probes and compliance checks that produce vulnerability reports, plus related products for guardrails, model security scanning and an MCP proxy.\n\nEvals execute on your machine and talk directly to the model provider, so no data passes through a hosted service unless you opt into sharing. The Community edition is free; Enterprise adds team sharing, SSO, continuous monitoring and managed cloud deployment, and an On-Premise edition deploys the same inside your infrastructure. promptfoo complements tracing platforms such as Langfuse rather than replacing them.",15  "capabilities": ["Declarative eval configs with prompts, providers and assertions", "Deterministic and model-graded scoring with caching and concurrency", "Side-by-side comparison in a local web viewer", "Automated red teaming and vulnerability reports", "Provider support for major APIs, local models and custom endpoints", "CLI, Node library and CI integration"],16  "workflows": ["Gate prompt changes in CI with a regression suite", "Compare two models on the same test set", "Red-team a chatbot before launch"],17  "advantages": ["Local-first; no data leaves your machine by default", "MIT license", "Covers both quality evals and security testing"],18  "limitations": ["Not a production tracing backend", "Model-graded assertions cost model calls", "Enterprise collaboration features are proprietary"],19  "prerequisites": ["Node.js", "API keys or local models for the providers under test"],20  "audiences": ["Developers testing prompts and agents", "Security teams red-teaming LLM apps"],21  "alternatives": ["ragas", "langfuse", "braintrust", "dspy"],22  "categories": [{ "slug": "llm-ops", "primary": true }, { "slug": "security-tools", "label": "LLM red teaming" }],23  "tags": ["llm", "testing", "cli", "open-source", "evaluation", "red-teaming", "typescript"],24  "use_cases": ["evaluate-llm-apps"],25  "operating_systems": ["macos", "linux", "windows"],26  "execution_modes": ["local", "self-hosted", "cloud"],27  "interfaces": ["cli", "library", "web"],28  "deployment_models": ["desktop", "on-premises", "saas"],29  "integrations": ["OpenAI", "Anthropic", "Azure OpenAI", "Google Gemini", "Hugging Face", "Ollama", "GitHub Actions"],30  "data_formats": ["YAML", "JSON", "CSV"],31  "languages": ["TypeScript", "JavaScript", "Python (custom providers)"],32  "source_model": "mixed",33  "license_spdx": "MIT",34  "pricing_model": ["open-source", "free", "enterprise"],35  "pricing_checked_at": "2026-11-02",36  "self_hostable": true,37  "sources": [38    { "url": "https://www.promptfoo.dev/", "type": "official_site", "title": "promptfoo", "supports": ["name", "summary"], "checked_at": "2026-11-02" },39    { "url": "https://www.promptfoo.dev/docs/intro/", "type": "official_docs", "title": "promptfoo introduction", "supports": ["capabilities", "integrations"], "checked_at": "2026-11-02" },40    { "url": "https://github.com/promptfoo/promptfoo/blob/main/LICENSE", "type": "license_page", "supports": ["license_spdx", "source_model"], "checked_at": "2026-11-02" },41    { "url": "https://www.promptfoo.dev/pricing/", "type": "pricing_page", "supports": ["pricing_model"], "checked_at": "2026-11-02", "note": "Community, Enterprise, On-Premise" }42  ],43  "confidence": "high",44  "discovery_method": "editorial-research",45  "depth": "standard",46  "editorial_status": "reviewed",47  "visibility": "public",48  "featured": false,49  "author": "batch-A"50}51