HTML 77.2%
TypeScript 10.5%
Python 9.6%
JavaScript 2.5%
1from datetime import date23from aiatlas.ids import kind_of, new_id, normalize_alias, slugify4from aiatlas.sdk.extract.dates import parse_date, parse_datetime5from aiatlas.sdk.extract.html import parse_html6from aiatlas.sdk.extract.markdown import parse_markdown7from aiatlas.sdk.extract.numbers import (8 parse_active_params,9 parse_context_length,10 parse_money_per_mtok,11 parse_param_count,12 parse_percent,13)14from aiatlas.sdk.fetch import canonicalize_url15from aiatlas.services.search import compile_query161718def test_ids_and_slugs():19 mid = new_id("model")20 assert mid.startswith("model_") and kind_of(mid) == "model"21 assert slugify("Qwen3-235B-A22B") == "qwen3-235b-a22b"22 assert slugify("GPT-4.1 mini") == "gpt-4.1-mini"23 assert normalize_alias("Claude 3.5 Haiku") == normalize_alias("claude-3-5-haiku") == "claude35haiku"242526def test_numbers():27 assert parse_param_count("Qwen3-235B-A22B") == 235_000_000_00028 assert parse_active_params("Qwen3-235B-A22B") == 22_000_000_00029 assert parse_param_count("a 7.6 billion parameters model") == 7_600_000_00030 assert parse_param_count("Llama 3.1 8B Instruct") == 8_000_000_00031 assert parse_param_count("version 2.0") is None32 assert parse_context_length("128K tokens") == 128_00033 assert parse_context_length("1M") == 1_000_00034 assert parse_context_length("200,000 tokens") == 200_00035 assert parse_context_length("32768") == 3276836 assert parse_money_per_mtok("$3.00 / 1M tokens") == 3.037 assert parse_money_per_mtok("$0.15/M") == 0.1538 assert parse_money_per_mtok("$2 per 1K tokens") == 2000.039 assert parse_percent("72.4%") == 72.4404142def test_dates():43 assert parse_datetime("2026-07-24T10:00:00Z").year == 202644 assert parse_date("July 24, 2026") == (date(2026, 7, 24), "day")45 assert parse_date("March 2025") == (date(2025, 3, 1), "month")46 assert parse_date("2024") == (date(2024, 1, 1), "year")47 assert parse_date("") == (None, "none")484950def test_canonical_url():51 assert canonicalize_url("https://Example.com/a/?utm_source=x&b=1#frag") == "https://example.com/a?b=1"52 assert canonicalize_url("https://example.com/") == "https://example.com/"535455def test_html_parse():56 html = """<html lang="en"><head><title>T</title><meta name="description" content="D"><link rel="canonical" href="/c">57 <script type="application/ld+json">{"@type":"Organization","name":"X"}</script>58 <script id="__NEXT_DATA__" type="application/json">{"props":{"models":[{"id":"m1"}]}}</script></head>59 <body><h1>Head</h1><table><tr><th>Model</th><th>Ctx</th></tr><tr><td>A</td><td>128K</td></tr></table><a href="/x">link</a><p>Body text</p></body></html>"""60 doc = parse_html(html, "https://example.com/p")61 assert doc.title == "T" and doc.description == "D" and doc.canonical == "https://example.com/c" and doc.lang == "en"62 assert doc.json_ld[0]["name"] == "X"63 assert doc.embedded_json["__NEXT_DATA__"]["props"]["models"][0]["id"] == "m1"64 assert doc.tables[0]["headers"] == ["Model", "Ctx"] and doc.tables[0]["rows"] == [["A", "128K"]]65 assert doc.links == [("https://example.com/x", "link")]66 assert "Body text" in doc.text and "Head" in doc.text676869def test_markdown_parse():70 md = parse_markdown("---\ntitle: X\nlicense: apache-2.0\n---\n# H1\n\n| a | b |\n|---|---|\n| 1 | 2 |\n\nSee [l](https://e.com).\n\n## Sec\ntext")71 assert md.front_matter == {"title": "X", "license": "apache-2.0"}72 assert md.tables[0]["headers"] == ["a", "b"] and md.tables[0]["rows"] == [["1", "2"]]73 assert md.links == [("https://e.com", "l")]74 assert md.section("Sec") == "text"757677def test_compile_query():78 q = compile_query("open models released in 2026 with more than 100B parameters and 128k context")79 assert q.entity_type == "model" and q.openness == "open" and q.year_from == 2026 and q.params_min == 100_000_000_000 and q.context_min == 128_00080 q2 = compile_query("vision models by Mistral")81 assert q2.entity_type == "model" and "image" in q2.modalities and q2.organization == "Mistral"82 assert compile_query("nvidia").filters["residual"] == "nvidia"83