spb/tools-spboucher-ai

1{2  "slug": "llama-cpp",3  "name": "llama.cpp",4  "aliases": ["llama-cpp", "ggml", "llama-server"],5  "vendor": { "name": "ggml-org (Georgi Gerganov and contributors)", "slug": "ggml-org", "website": "https://github.com/ggml-org" },6  "homepage_url": "https://github.com/ggml-org/llama.cpp",7  "docs_url": "https://github.com/ggml-org/llama.cpp",8  "repo_url": "https://github.com/ggml-org/llama.cpp",9  "license_url": "https://github.com/ggml-org/llama.cpp/blob/master/LICENSE",10  "product_type": "library",11  "product_status": "active",12  "summary": "C/C++ inference engine for LLMs and vision-language models with GGUF quantized weights, Metal, CUDA, Vulkan and other backends, a CLI and an OpenAI-compatible server",13  "description": "llama.cpp is an MIT-licensed C/C++ engine for running large language models, and increasingly vision-language models, on commodity hardware. It began in 2023 as a port of Meta's LLaMA to Apple Silicon and grew into the reference implementation behind the GGUF file format and the GGML tensor library; the repository moved from ggerganov to the ggml-org organization. Models are converted from Hugging Face checkpoints with the bundled `convert_hf_to_gguf.py` script and quantized to integer formats from 1.5 to 8 bits, which lets multi-billion-parameter models fit in laptop memory.\n\nHardware coverage is unusually wide: ARM NEON, Accelerate and Metal on Apple Silicon; AVX variants and AMX on x86; CUDA, HIP, MUSA, Vulkan, SYCL, OpenCL, WebGPU, CANN and other accelerators; plus hybrid CPU-GPU offload when a model exceeds VRAM. The project ships a CLI (`llama cli`) that can download models directly from Hugging Face, a `completion` tool with GBNF grammar-constrained output, and `llama-server`, an OpenAI-compatible HTTP server with a built-in web UI. Installation is a script, Docker image, prebuilt release or source build, and bindings exist in the wider ecosystem for Python, Go, Rust, Node and others.\n\nllama.cpp is the engine inside Ollama, LM Studio and many other front ends, so it is the layer to reach for when you need the newest model architectures, fine-grained control of quantization and sampling, or a minimal dependency footprint on a server. Its limits are that it is a single-node engine optimized for latency on one or a few requests rather than for high-concurrency batching, that model support depends on community conversion work, and that the fast release cadence occasionally changes CLI flags and file layouts.",14  "capabilities": ["GGUF model format with integer quantization from 1.5- to 8-bit", "Backends for Metal, CUDA, HIP, Vulkan, SYCL, OpenCL, WebGPU and CPU SIMD", "Hybrid CPU+GPU offload for models larger than VRAM", "`llama-server` OpenAI-compatible API with web UI", "CLI with direct Hugging Face downloads and grammar-constrained generation", "Multimodal (vision-language) inference", "Conversion scripts from Hugging Face checkpoints", "Speculative decoding and KV-cache controls"],15  "workflows": ["Convert and quantize a Hugging Face model to GGUF for a Mac Studio", "Run `llama-server` as a local OpenAI-compatible endpoint for an application", "Benchmark quantization levels against quality on a given machine", "Embed the C API inside a native application"],16  "advantages": ["Minimal dependencies and portable C/C++ codebase", "Fastest adoption of new open-weight architectures", "Runs on hardware from Raspberry Pi to multi-GPU servers", "Permissive MIT license"],17  "limitations": ["Not designed for high-concurrency serving", "Requires model conversion to GGUF", "Frequent breaking changes in CLI and build options", "Quality loss at aggressive quantization levels"],18  "prerequisites": ["C/C++ toolchain or a prebuilt release; GPU drivers for accelerated backends", "Python for conversion scripts"],19  "audiences": ["Developers embedding local inference", "Researchers experimenting with quantization", "Operators of edge and on-premises inference"],20  "alternatives": ["ollama", "lm-studio", "vllm", "mlx-lm"],21  "comparison_notes": "Ollama and LM Studio wrap llama.cpp with model management and friendlier interfaces, trading control for convenience. vLLM is the server-grade alternative with PagedAttention and continuous batching for many concurrent users on data-center GPUs. mlx-lm is Apple's framework-native option on Apple Silicon and uses its own weight format instead of GGUF.",22  "categories": [{ "slug": "local-inference", "primary": true }, { "slug": "apple-silicon-ml", "label": "Metal backend and origin on Apple Silicon" }],23  "tags": ["llm", "open-source", "self-hosted", "cli", "api", "apple-silicon", "quantization", "gguf"],24  "use_cases": ["run-local-models", "serve-models"],25  "operating_systems": ["macos", "linux", "windows"],26  "execution_modes": ["local", "self-hosted"],27  "interfaces": ["cli", "api", "library"],28  "deployment_models": ["embedded", "on-premises"],29  "integrations": ["Ollama", "LM Studio", "Hugging Face Hub", "Docker", "llama-cpp-python"],30  "data_formats": ["GGUF", "Safetensors (conversion input)", "GBNF grammars"],31  "languages": ["C", "C++", "Python (tooling)"],32  "source_model": "open-source",33  "license_spdx": "MIT",34  "pricing_model": ["open-source", "free"],35  "self_hostable": true,36  "sources": [37    { "url": "https://github.com/ggml-org/llama.cpp", "type": "official_site", "title": "ggml-org/llama.cpp", "supports": ["name", "capabilities", "operating_systems", "interfaces"], "checked_at": "2026-11-02" },38    { "url": "https://github.com/ggml-org/llama.cpp", "type": "repository", "supports": ["languages", "license_spdx"], "checked_at": "2026-11-02", "note": "Repository metadata reports MIT; former URL github.com/ggerganov/llama.cpp redirects here" },39    { "url": "https://github.com/ggml-org/llama.cpp/blob/master/LICENSE", "type": "license_page", "supports": ["license_spdx", "source_model"], "checked_at": "2026-11-02" }40  ],41  "confidence": "high",42  "discovery_method": "editorial-research",43  "depth": "deep",44  "editorial_status": "reviewed",45  "visibility": "public",46  "featured": false,47  "author": "batch-A"48}49