1from datetime import date23from aiatlas.ids import kind_of, new_id, normalize_alias, slugify4from aiatlas.sdk.extract.dates import parse_date, parse_datetime5from aiatlas.sdk.extract.html import parse_html6from aiatlas.sdk.extract.markdown import parse_markdown7from aiatlas.sdk.extract.numbers import parse_active_params, parse_context_length, parse_money_per_mtok, parse_param_count, parse_percent8from aiatlas.sdk.fetch import canonicalize_url9from aiatlas.services.search import compile_query101112def test_ids_and_slugs():13 mid = new_id("model")14 assert mid.startswith("model_") and kind_of(mid) == "model"15 assert slugify("Qwen3-235B-A22B") == "qwen3-235b-a22b"16 assert slugify("GPT-4.1 mini") == "gpt-4.1-mini"17 assert normalize_alias("Claude 3.5 Haiku") == normalize_alias("claude-3-5-haiku") == "claude35haiku"181920def test_numbers():21 assert parse_param_count("Qwen3-235B-A22B") == 235_000_000_00022 assert parse_active_params("Qwen3-235B-A22B") == 22_000_000_00023 assert parse_param_count("a 7.6 billion parameters model") == 7_600_000_00024 assert parse_param_count("Llama 3.1 8B Instruct") == 8_000_000_00025 assert parse_param_count("version 2.0") is None26 assert parse_context_length("128K tokens") == 128_00027 assert parse_context_length("1M") == 1_000_00028 assert parse_context_length("200,000 tokens") == 200_00029 assert parse_context_length("32768") == 3276830 assert parse_money_per_mtok("$3.00 / 1M tokens") == 3.031 assert parse_money_per_mtok("$0.15/M") == 0.1532 assert parse_money_per_mtok("$2 per 1K tokens") == 2000.033 assert parse_percent("72.4%") == 72.4343536def test_dates():37 assert parse_datetime("2026-07-24T10:00:00Z").year == 202638 assert parse_date("July 24, 2026") == (date(2026, 7, 24), "day")39 assert parse_date("March 2025") == (date(2025, 3, 1), "month")40 assert parse_date("2024") == (date(2024, 1, 1), "year")41 assert parse_date("") == (None, "none")424344def test_canonical_url():45 assert canonicalize_url("https://Example.com/a/?utm_source=x&b=1#frag") == "https://example.com/a?b=1"46 assert canonicalize_url("https://example.com/") == "https://example.com/"474849def test_html_parse():50 html = """<html lang="en"><head><title>T</title><meta name="description" content="D"><link rel="canonical" href="/c">51 <script type="application/ld+json">{"@type":"Organization","name":"X"}</script>52 <script id="__NEXT_DATA__" type="application/json">{"props":{"models":[{"id":"m1"}]}}</script></head>53 <body><h1>Head</h1><table><tr><th>Model</th><th>Ctx</th></tr><tr><td>A</td><td>128K</td></tr></table><a href="/x">link</a><p>Body text</p></body></html>"""54 doc = parse_html(html, "https://example.com/p")55 assert doc.title == "T" and doc.description == "D" and doc.canonical == "https://example.com/c" and doc.lang == "en"56 assert doc.json_ld[0]["name"] == "X"57 assert doc.embedded_json["__NEXT_DATA__"]["props"]["models"][0]["id"] == "m1"58 assert doc.tables[0]["headers"] == ["Model", "Ctx"] and doc.tables[0]["rows"] == [["A", "128K"]]59 assert doc.links == [("https://example.com/x", "link")]60 assert "Body text" in doc.text and "Head" in doc.text616263def test_markdown_parse():64 md = parse_markdown("---\ntitle: X\nlicense: apache-2.0\n---\n# H1\n\n| a | b |\n|---|---|\n| 1 | 2 |\n\nSee [l](https://e.com).\n\n## Sec\ntext")65 assert md.front_matter == {"title": "X", "license": "apache-2.0"}66 assert md.tables[0]["headers"] == ["a", "b"] and md.tables[0]["rows"] == [["1", "2"]]67 assert md.links == [("https://e.com", "l")]68 assert md.section("Sec") == "text"697071def test_compile_query():72 q = compile_query("open models released in 2026 with more than 100B parameters and 128k context")73 assert q.entity_type == "model" and q.openness == "open" and q.year_from == 2026 and q.params_min == 100_000_000_000 and q.context_min == 128_00074 q2 = compile_query("vision models by Mistral")75 assert q2.entity_type == "model" and "image" in q2.modalities and q2.organization == "Mistral"76 assert compile_query("nvidia").filters["residual"] == "nvidia"77