#!/usr/bin/env python3
"""Probe the long tail of SaaS / B2B / consumer-internet services for machine-readable status pages,
changelog / release-note feeds and server-rendered changelog / pricing pages, then emit a registry
fragment to curate (config/sources.d/48-saas-status-changelogs.yaml).

Usage:
  python3 scripts/gen-status-probe.py probe  [--out /tmp/probe48.jsonl] [--only domain,domain]
  python3 scripts/gen-status-probe.py emit   [--in /tmp/probe48.jsonl] > config/sources.d/48-saas-status-changelogs.yaml
Then validate with apps/engine/src/validate.ts and prune with scripts/prune-fragment.py.

Politeness: one global token bucket (≤ 4 requests/s), 15 s timeout, ≤ 1.5 MB per body, UA
"WebSensor registry probe (contact@websensor.io)". Hosts that do not resolve cost no request.
Duplicates: URLs already present anywhere in config/ are never emitted; status hosts already covered by
an existing STATUSPAGE sensor are not probed again; ids that exist in an earlier file are `extend: true`.
"""
from __future__ import annotations

import argparse
import glob
import hashlib
import json
import os
import re
import socket
import sys
import threading
import time
from concurrent.futures import ThreadPoolExecutor, as_completed
from urllib.parse import urlparse

import requests
import yaml

UA = "WebSensor registry probe (contact@websensor.io)"
TIMEOUT = 15
MAX_BODY = 1_500_000
RATE = 4.0  # requests per second, global

# ───────────────────────── candidates: domain | Name | categories | country (default US) ─────────────────────────
CANDIDATES = """
# collaboration & productivity
slack.com | Slack | enterprise,saas
zoom.us | Zoom | enterprise,saas
notion.so | Notion | enterprise,saas
figma.com | Figma | enterprise,saas,developer
miro.com | Miro | enterprise,saas
loom.com | Loom | enterprise,saas
canva.com | Canva | consumer-tech,saas | AU
airtable.com | Airtable | enterprise,saas
asana.com | Asana | enterprise,saas
monday.com | monday.com | enterprise,saas | IL
clickup.com | ClickUp | enterprise,saas
linear.app | Linear | developer,saas
basecamp.com | Basecamp | enterprise,saas
trello.com | Trello | enterprise,saas
coda.io | Coda | enterprise,saas
dropbox.com | Dropbox | enterprise,consumer-tech
box.com | Box | enterprise,saas
zapier.com | Zapier | enterprise,saas
make.com | Make | enterprise,saas | CZ
n8n.io | n8n | developer,open-source,saas | DE
calendly.com | Calendly | enterprise,saas
cal.com | Cal.com | developer,open-source,saas
typeform.com | Typeform | enterprise,saas | ES
surveymonkey.com | SurveyMonkey | enterprise,saas
qualtrics.com | Qualtrics | enterprise,saas
evernote.com | Evernote | consumer-tech,saas
todoist.com | Todoist | consumer-tech,saas
ticktick.com | TickTick | consumer-tech,saas
obsidian.md | Obsidian | consumer-tech,developer
craft.do | Craft | consumer-tech,saas
whimsical.com | Whimsical | enterprise,saas
lucid.co | Lucid Software | enterprise,saas
mural.co | Mural | enterprise,saas
pitch.com | Pitch | enterprise,saas | DE
gamma.app | Gamma | enterprise,saas,ai
tldraw.com | tldraw | developer,open-source
excalidraw.com | Excalidraw | developer,open-source
docusign.com | Docusign | enterprise,saas,legal
hellosign.com | Dropbox Sign | enterprise,saas
pandadoc.com | PandaDoc | enterprise,saas
ironcladapp.com | Ironclad | enterprise,saas,legal
1password.com | 1Password | cyber,saas | CA
lastpass.com | LastPass | cyber,saas
bitwarden.com | Bitwarden | cyber,open-source,saas
dashlane.com | Dashlane | cyber,saas | FR
keeper.io | Keeper Security | cyber,saas
grammarly.com | Grammarly | consumer-tech,saas,ai
otter.ai | Otter.ai | enterprise,saas,ai
fireflies.ai | Fireflies.ai | enterprise,saas,ai
gong.io | Gong | enterprise,saas | IL
clari.com | Clari | enterprise,saas
outreach.io | Outreach | enterprise,saas
salesloft.com | Salesloft | enterprise,saas
apollo.io | Apollo.io | enterprise,saas
zoominfo.com | ZoomInfo | enterprise,saas
lusha.com | Lusha | enterprise,saas | IL
clearbit.com | Clearbit | enterprise,saas
drift.com | Drift | enterprise,saas
front.com | Front | enterprise,saas
helpscout.com | Help Scout | enterprise,saas
gorgias.com | Gorgias | enterprise,saas,commerce
kustomer.com | Kustomer | enterprise,saas
crisp.chat | Crisp | enterprise,saas | FR
tawk.to | tawk.to | enterprise,saas
livechat.com | LiveChat | enterprise,saas | PL
text.com | Text (LiveChat) | enterprise,saas | PL
tidio.com | Tidio | enterprise,saas | PL
olark.com | Olark | enterprise,saas
dialpad.com | Dialpad | enterprise,telecom,saas
aircall.io | Aircall | enterprise,telecom,saas | FR
ringcentral.com | RingCentral | enterprise,telecom,saas
8x8.com | 8x8 | enterprise,telecom,saas
webex.com | Webex | enterprise,telecom,saas
goto.com | GoTo | enterprise,telecom,saas
whereby.com | Whereby | enterprise,saas | NO
teamviewer.com | TeamViewer | enterprise,saas | DE
anydesk.com | AnyDesk | enterprise,saas | DE
splashtop.com | Splashtop | enterprise,saas
smartsheet.com | Smartsheet | enterprise,saas
wrike.com | Wrike | enterprise,saas
teamwork.com | Teamwork.com | enterprise,saas | IE
productboard.com | Productboard | enterprise,saas | CZ
aha.io | Aha! | enterprise,saas
canny.io | Canny | enterprise,saas
featurebase.app | Featurebase | enterprise,saas
shortcut.com | Shortcut | developer,saas
height.app | Height | developer,saas
plane.so | Plane | developer,open-source,saas
fibery.io | Fibery | enterprise,saas | CY
lattice.com | Lattice | enterprise,saas
cultureamp.com | Culture Amp | enterprise,saas | AU
15five.com | 15Five | enterprise,saas
leapsome.com | Leapsome | enterprise,saas | DE
workday.com | Workday | enterprise,saas
bamboohr.com | BambooHR | enterprise,saas
gusto.com | Gusto | enterprise,payments,saas
rippling.com | Rippling | enterprise,saas
deel.com | Deel | enterprise,labour,saas
remote.com | Remote | enterprise,labour,saas
justworks.com | Justworks | enterprise,saas
trinet.com | TriNet | enterprise,saas
paychex.com | Paychex | enterprise,payments,saas
paylocity.com | Paylocity | enterprise,saas
paycom.com | Paycom | enterprise,saas
personio.de | Personio | enterprise,saas | DE
hibob.com | HiBob | enterprise,saas | IL
greenhouse.io | Greenhouse | enterprise,labour,saas
lever.co | Lever | enterprise,labour,saas
ashbyhq.com | Ashby | enterprise,labour,saas
workable.com | Workable | enterprise,labour,saas
smartrecruiters.com | SmartRecruiters | enterprise,labour,saas
icims.com | iCIMS | enterprise,labour,saas
checkr.com | Checkr | enterprise,saas
vanta.com | Vanta | cyber,enterprise,saas
drata.com | Drata | cyber,enterprise,saas
secureframe.com | Secureframe | cyber,enterprise,saas
dropboxforum.com | Dropbox Forum | enterprise
intercom.com | Intercom | enterprise,saas
zendesk.com | Zendesk | enterprise,saas
freshworks.com | Freshworks | enterprise,saas
hubspot.com | HubSpot | enterprise,saas
pipedrive.com | Pipedrive | enterprise,saas | EE
zoho.com | Zoho | enterprise,saas | IN
mailchimp.com | Mailchimp | enterprise,saas
klaviyo.com | Klaviyo | enterprise,commerce,saas
braze.com | Braze | enterprise,saas
iterable.com | Iterable | enterprise,saas
customer.io | Customer.io | enterprise,saas
segment.com | Twilio Segment | developer,saas
amplitude.com | Amplitude | enterprise,saas
mixpanel.com | Mixpanel | enterprise,saas
heap.io | Heap | enterprise,saas
hotjar.com | Hotjar | enterprise,saas | MT
fullstory.com | Fullstory | enterprise,saas
launchdarkly.com | LaunchDarkly | developer,saas
optimizely.com | Optimizely | enterprise,saas
split.io | Split | developer,saas
statsig.com | Statsig | developer,saas
posthog.com | PostHog | developer,open-source,saas
rollbar.com | Rollbar | developer,saas
bugsnag.com | Bugsnag | developer,saas
opsgenie.com | Opsgenie | developer,saas
firehydrant.com | FireHydrant | developer,saas
incident.io | incident.io | developer,saas | GB
statuspage.io | Atlassian Statuspage | developer,saas
# developer platforms & APIs
clerk.com | Clerk | developer,saas
workos.com | WorkOS | developer,saas
stytch.com | Stytch | developer,saas
descope.com | Descope | developer,saas
frontegg.com | Frontegg | developer,saas | IL
supertokens.com | SuperTokens | developer,open-source
postmarkapp.com | Postmark | developer,saas
mailgun.com | Mailgun | developer,saas
mailtrap.io | Mailtrap | developer,saas
mailjet.com | Mailjet | developer,saas | FR
sparkpost.com | SparkPost | developer,saas
courier.com | Courier | developer,saas
knock.app | Knock | developer,saas
onesignal.com | OneSignal | developer,saas
novu.co | Novu | developer,open-source,saas
algolia.com | Algolia | developer,saas | FR
typesense.org | Typesense | developer,open-source
meilisearch.com | Meilisearch | developer,open-source,saas | FR
cloudinary.com | Cloudinary | developer,saas | IL
imgix.com | imgix | developer,saas
uploadcare.com | Uploadcare | developer,saas
filestack.com | Filestack | developer,saas
mux.com | Mux | developer,saas
vimeo.com | Vimeo | media,saas
wistia.com | Wistia | enterprise,media,saas
bitmovin.com | Bitmovin | developer,media,saas | AT
agora.io | Agora | developer,telecom,saas
daily.co | Daily | developer,telecom,saas
livekit.io | LiveKit | developer,open-source,telecom
100ms.live | 100ms | developer,telecom,saas | IN
ably.com | Ably | developer,saas | GB
pusher.com | Pusher | developer,saas | GB
pubnub.com | PubNub | developer,saas
getstream.io | Stream | developer,saas
sendbird.com | Sendbird | developer,saas
vonage.com | Vonage | developer,telecom,saas
bird.com | Bird | developer,telecom,saas | NL
plivo.com | Plivo | developer,telecom,saas
telnyx.com | Telnyx | developer,telecom,saas
sinch.com | Sinch | developer,telecom,saas | SE
bandwidth.com | Bandwidth | developer,telecom,saas
infobip.com | Infobip | developer,telecom,saas | HR
postman.com | Postman | developer,saas
readme.com | ReadMe | developer,saas
mintlify.com | Mintlify | developer,saas
gitbook.com | GitBook | developer,saas
swagger.io | Swagger | developer,open-source
stoplight.io | Stoplight | developer,saas
konghq.com | Kong | developer,open-source,saas
tyk.io | Tyk | developer,open-source,saas | GB
zuplo.com | Zuplo | developer,saas
hookdeck.com | Hookdeck | developer,saas | CA
svix.com | Svix | developer,saas
ngrok.com | ngrok | developer,saas
goteleport.com | Teleport | developer,cyber,saas
doppler.com | Doppler | developer,cyber,saas
infisical.com | Infisical | developer,open-source,cyber
pulumi.com | Pulumi | developer,saas
spacelift.io | Spacelift | developer,saas
env0.com | env0 | developer,saas | IL
scalr.com | Scalr | developer,saas
circleci.com | CircleCI | developer,saas
buildkite.com | Buildkite | developer,saas | AU
travis-ci.com | Travis CI | developer,saas
semaphoreci.com | Semaphore CI | developer,saas
codemagic.io | Codemagic | developer,saas | EE
bitrise.io | Bitrise | developer,saas | HU
expo.dev | Expo | developer,open-source,saas
harness.io | Harness | developer,saas
octopus.com | Octopus Deploy | developer,saas | AU
codecov.io | Codecov | developer,saas
coveralls.io | Coveralls | developer,saas
sonarsource.com | SonarSource | developer,saas | CH
snyk.io | Snyk | cyber,developer,saas
mend.io | Mend.io | cyber,developer,saas | IL
jfrog.com | JFrog | developer,saas | IL
sonatype.com | Sonatype | developer,cyber,saas
cloudsmith.com | Cloudsmith | developer,saas | GB
replit.com | Replit | developer,saas,ai
codesandbox.io | CodeSandbox | developer,saas
stackblitz.com | StackBlitz | developer,saas
gitpod.io | Gitpod | developer,saas | CH
coder.com | Coder | developer,open-source,saas
cursor.com | Cursor | developer,ai,saas
windsurf.com | Windsurf | developer,ai,saas
sourcegraph.com | Sourcegraph | developer,saas
warp.dev | Warp | developer,saas
raycast.com | Raycast | developer,consumer-tech
jetbrains.com | JetBrains | developer,software | CZ
orbstack.dev | OrbStack | developer,software
localstack.cloud | LocalStack | developer,saas | CH
browserstack.com | BrowserStack | developer,saas | IN
saucelabs.com | Sauce Labs | developer,saas
lambdatest.com | LambdaTest | developer,saas | IN
cypress.io | Cypress | developer,open-source,saas
checklyhq.com | Checkly | developer,saas | DE
honeycomb.io | Honeycomb | developer,saas
chronosphere.io | Chronosphere | developer,saas
betterstack.com | Better Stack | developer,saas | CZ
uptimerobot.com | UptimeRobot | developer,saas | MT
pingdom.com | Pingdom | developer,saas
statuscake.com | StatusCake | developer,saas | GB
cronitor.io | Cronitor | developer,saas
healthchecks.io | Healthchecks.io | developer,open-source,saas
squadcast.com | Squadcast | developer,saas
rootly.com | Rootly | developer,saas
splunk.com | Splunk | enterprise,cyber,saas
sumologic.com | Sumo Logic | enterprise,developer,saas
logz.io | Logz.io | developer,saas | IL
axiom.co | Axiom | developer,saas
highlight.io | Highlight.io | developer,open-source,saas
logrocket.com | LogRocket | developer,saas
smartlook.com | Smartlook | enterprise,saas | CZ
mouseflow.com | Mouseflow | enterprise,saas | DK
contentsquare.com | Contentsquare | enterprise,saas | FR
pendo.io | Pendo | enterprise,saas
appcues.com | Appcues | enterprise,saas
userpilot.com | Userpilot | enterprise,saas
chameleon.io | Chameleon | enterprise,saas
walkme.com | WalkMe | enterprise,saas | IL
whatfix.com | Whatfix | enterprise,saas | IN
contentful.com | Contentful | developer,saas | DE
sanity.io | Sanity | developer,saas | NO
strapi.io | Strapi | developer,open-source,saas | FR
storyblok.com | Storyblok | developer,saas | AT
hygraph.com | Hygraph | developer,saas | DE
prismic.io | Prismic | developer,saas | FR
directus.io | Directus | developer,open-source,saas
payloadcms.com | Payload | developer,open-source
builder.io | Builder.io | developer,saas
plasmic.app | Plasmic | developer,saas
webflow.com | Webflow | developer,saas
framer.com | Framer | developer,saas | NL
squarespace.com | Squarespace | commerce,saas
wix.com | Wix | commerce,saas | IL
wordpress.com | WordPress.com | media,saas
ghost.org | Ghost | media,open-source,saas
substack.com | Substack | media,saas
medium.com | Medium | media,saas
beehiiv.com | beehiiv | media,saas
kit.com | Kit (ConvertKit) | media,saas
buttondown.com | Buttondown | media,saas
mailerlite.com | MailerLite | enterprise,saas | LT
activecampaign.com | ActiveCampaign | enterprise,saas
constantcontact.com | Constant Contact | enterprise,saas
campaignmonitor.com | Campaign Monitor | enterprise,saas | AU
brevo.com | Brevo | enterprise,saas | FR
getresponse.com | GetResponse | enterprise,saas | PL
omnisend.com | Omnisend | enterprise,commerce,saas | LT
attentive.com | Attentive | enterprise,commerce,saas
postscript.io | Postscript | enterprise,commerce,saas
yotpo.com | Yotpo | enterprise,commerce,saas | IL
rechargepayments.com | Recharge | commerce,payments,saas
sproutsocial.com | Sprout Social | enterprise,saas
hootsuite.com | Hootsuite | enterprise,saas | CA
buffer.com | Buffer | enterprise,saas
later.com | Later | enterprise,saas | CA
sprinklr.com | Sprinklr | enterprise,saas
brandwatch.com | Brandwatch | enterprise,saas | GB
meltwater.com | Meltwater | enterprise,saas | NO
semrush.com | Semrush | enterprise,saas
ahrefs.com | Ahrefs | enterprise,saas | SG
moz.com | Moz | enterprise,saas
similarweb.com | Similarweb | enterprise,saas | IL
crazyegg.com | Crazy Egg | enterprise,saas
vwo.com | VWO | enterprise,saas | IN
abtasty.com | AB Tasty | enterprise,saas | FR
unbounce.com | Unbounce | enterprise,saas | CA
instapage.com | Instapage | enterprise,saas
carrd.co | Carrd | consumer-tech,saas
linktr.ee | Linktree | consumer-tech,saas | AU
bitly.com | Bitly | enterprise,saas
dub.co | Dub | developer,saas
short.io | Short.io | enterprise,saas
savvycal.com | SavvyCal | enterprise,saas
doodle.com | Doodle | consumer-tech,saas | CH
# data, analytics & internal tools
getdbt.com | dbt Labs | developer,saas
fivetran.com | Fivetran | developer,saas
airbyte.com | Airbyte | developer,open-source,saas
stitchdata.com | Stitch | developer,saas
hightouch.com | Hightouch | developer,saas
getcensus.com | Census | developer,saas
looker.com | Looker | enterprise,saas
tableau.com | Tableau | enterprise,saas
metabase.com | Metabase | developer,open-source,saas
mode.com | Mode | enterprise,saas
hex.tech | Hex | developer,saas
observablehq.com | Observable | developer,saas
retool.com | Retool | developer,saas
appsmith.com | Appsmith | developer,open-source,saas
bubble.io | Bubble | developer,saas
tooljet.com | ToolJet | developer,open-source,saas
budibase.com | Budibase | developer,open-source,saas | GB
softr.io | Softr | developer,saas | DE
glideapps.com | Glide | developer,saas
adalo.com | Adalo | developer,saas
flutterflow.io | FlutterFlow | developer,saas
workato.com | Workato | enterprise,saas
tray.io | Tray.ai | enterprise,saas
celigo.com | Celigo | enterprise,saas
boomi.com | Boomi | enterprise,saas
informatica.com | Informatica | enterprise,saas
matillion.com | Matillion | developer,saas | GB
alteryx.com | Alteryx | enterprise,saas
dataiku.com | Dataiku | enterprise,ai,saas | FR
domo.com | Domo | enterprise,saas
sisense.com | Sisense | enterprise,saas
thoughtspot.com | ThoughtSpot | enterprise,saas
qlik.com | Qlik | enterprise,saas
microstrategy.com | Strategy (MicroStrategy) | enterprise,saas
gooddata.com | GoodData | enterprise,saas
cube.dev | Cube | developer,open-source,saas
preset.io | Preset | developer,saas
lightdash.com | Lightdash | developer,open-source,saas | GB
evidence.dev | Evidence | developer,open-source
count.co | Count | developer,saas | GB
deepnote.com | Deepnote | developer,saas
anaconda.com | Anaconda | developer,software
comet.com | Comet | developer,ai,saas
neptune.ai | Neptune.ai | developer,ai,saas | PL
tecton.ai | Tecton | developer,ai,saas
prefect.io | Prefect | developer,open-source,saas
dagster.io | Dagster | developer,open-source,saas
astronomer.io | Astronomer | developer,saas
mage.ai | Mage | developer,open-source,saas
kestra.io | Kestra | developer,open-source,saas | FR
temporal.io | Temporal | developer,open-source,saas
trigger.dev | Trigger.dev | developer,open-source,saas
windmill.dev | Windmill | developer,open-source,saas
pipedream.com | Pipedream | developer,saas
val.town | Val Town | developer,saas
montecarlodata.com | Monte Carlo | developer,saas
bigeye.com | Bigeye | developer,saas
soda.io | Soda | developer,saas | BE
atlan.com | Atlan | enterprise,saas | SG
alation.com | Alation | enterprise,saas
collibra.com | Collibra | enterprise,saas | BE
secoda.co | Secoda | enterprise,saas | CA
firebolt.io | Firebolt | developer,saas | IL
motherduck.com | MotherDuck | developer,saas
starburst.io | Starburst | developer,saas
dremio.com | Dremio | developer,saas
imply.io | Imply | developer,saas
timescale.com | Timescale | developer,open-source,saas
influxdata.com | InfluxData | developer,open-source,saas
questdb.io | QuestDB | developer,open-source,saas | GB
singlestore.com | SingleStore | developer,saas
planetscale.com | PlanetScale | developer,saas
yugabyte.com | YugabyteDB | developer,open-source,saas
fauna.com | Fauna | developer,saas
xata.io | Xata | developer,saas | DE
turso.tech | Turso | developer,saas
nhost.io | Nhost | developer,open-source,saas | SE
appwrite.io | Appwrite | developer,open-source,saas
convex.dev | Convex | developer,saas
hasura.io | Hasura | developer,open-source,saas
prisma.io | Prisma | developer,open-source,saas | DE
apollographql.com | Apollo GraphQL | developer,open-source,saas
gomomento.com | Momento | developer,saas
weaviate.io | Weaviate | developer,open-source,ai,saas | NL
qdrant.tech | Qdrant | developer,open-source,ai,saas | DE
zilliz.com | Zilliz | developer,ai,saas
lancedb.com | LanceDB | developer,open-source,ai
vespa.ai | Vespa.ai | developer,open-source,ai | NO
turbopuffer.com | turbopuffer | developer,ai,saas | CA
opensearch.org | OpenSearch | developer,open-source
constructor.io | Constructor | commerce,saas
bloomreach.com | Bloomreach | commerce,saas
# payments, fintech & finance apps
adyen.com | Adyen | payments,finance | NL
checkout.com | Checkout.com | payments,finance | GB
braintreepayments.com | Braintree | payments,developer
squareup.com | Square | payments,commerce
recurly.com | Recurly | payments,saas
chargebee.com | Chargebee | payments,saas
paddle.com | Paddle | payments,saas | GB
lemonsqueezy.com | Lemon Squeezy | payments,saas
gumroad.com | Gumroad | commerce,payments
fastspring.com | FastSpring | payments,saas
gocardless.com | GoCardless | payments,finance | GB
unit.co | Unit | payments,developer
moderntreasury.com | Modern Treasury | payments,developer
ramp.com | Ramp | finance,saas
revolut.com | Revolut | finance,payments | GB
n26.com | N26 | finance | DE
monzo.com | Monzo | finance | GB
starlingbank.com | Starling Bank | finance | GB
sofi.com | SoFi | finance
afterpay.com | Afterpay | payments | AU
sezzle.com | Sezzle | payments
zip.co | Zip | payments | AU
bolt.com | Bolt | payments,commerce
venmo.com | Venmo | payments,consumer-tech
dwolla.com | Dwolla | payments,developer
tipalti.com | Tipalti | payments,saas
bill.com | BILL | payments,saas
melio.com | Melio | payments,saas
navan.com | Navan | enterprise,travel,saas
spendesk.com | Spendesk | finance,saas | FR
pleo.io | Pleo | finance,saas | DK
payhawk.com | Payhawk | finance,saas | GB
coupa.com | Coupa | enterprise,saas
intuit.com | Intuit | finance,saas
quickbooks.intuit.com | QuickBooks | finance,saas
zuora.com | Zuora | payments,saas
maxio.com | Maxio | payments,saas
withorb.com | Orb | payments,developer
metronome.com | Metronome | payments,developer
getlago.com | Lago | payments,open-source,developer | FR
stigg.io | Stigg | payments,developer | IL
revenuecat.com | RevenueCat | payments,developer
adapty.io | Adapty | payments,developer
avalara.com | Avalara | finance,saas
taxjar.com | TaxJar | finance,saas
anrok.com | Anrok | finance,saas
alloy.com | Alloy | finance,cyber,saas
withpersona.com | Persona | cyber,saas
onfido.com | Onfido | cyber,saas | GB
jumio.com | Jumio | cyber,saas
veriff.com | Veriff | cyber,saas | EE
socure.com | Socure | cyber,saas
sardine.ai | Sardine | cyber,payments,saas
sift.com | Sift | cyber,saas
forter.com | Forter | cyber,commerce,saas | IL
riskified.com | Riskified | cyber,commerce,saas | IL
signifyd.com | Signifyd | cyber,commerce,saas
unit21.ai | Unit21 | cyber,finance,saas
chainalysis.com | Chainalysis | crypto,cyber,saas
elliptic.co | Elliptic | crypto,cyber,saas | GB
etoro.com | eToro | finance | IL
interactivebrokers.com | Interactive Brokers | finance
schwab.com | Charles Schwab | finance
fidelity.com | Fidelity Investments | finance
vanguard.com | Vanguard | finance
questrade.com | Questrade | finance | CA
betterment.com | Betterment | finance
wealthfront.com | Wealthfront | finance
acorns.com | Acorns | finance
public.com | Public | finance
alpaca.markets | Alpaca | finance,developer
tradier.com | Tradier | finance,developer
polygon.io | Polygon.io | finance,developer
finnhub.io | Finnhub | finance,developer
twelvedata.com | Twelve Data | finance,developer
intrinio.com | Intrinio | finance,developer
financialmodelingprep.com | Financial Modeling Prep | finance,developer
ynab.com | YNAB | finance,consumer-tech
nerdwallet.com | NerdWallet | finance,media
koho.ca | KOHO | finance | CA
neofinancial.com | Neo Financial | finance | CA
borrowell.com | Borrowell | finance | CA
razorpay.com | Razorpay | payments | IN
paytm.com | Paytm | payments | IN
phonepe.com | PhonePe | payments | IN
nuvei.com | Nuvei | payments | CA
paymentus.com | Paymentus | payments | CA
plooto.com | Plooto | payments | CA
clio.com | Clio | enterprise,legal,saas | CA
jobber.com | Jobber | enterprise,saas | CA
benevity.com | Benevity | enterprise,saas | CA
ada.cx | Ada | enterprise,ai,saas | CA
kinaxis.com | Kinaxis | enterprise,saas | CA
league.com | League | health,saas | CA
tophat.com | Top Hat | education,saas | CA
faire.com | Faire | commerce
# consumer internet
spotify.com | Spotify | entertainment,media | SE
netflix.com | Netflix | entertainment,media
disneyplus.com | Disney+ | entertainment,media
youtube.com | YouTube | media,internet
twitch.tv | Twitch | media,gaming
discord.com | Discord | consumer-tech,gaming
pinterest.com | Pinterest | consumer-tech,internet
snap.com | Snap | consumer-tech,internet
tiktok.com | TikTok | consumer-tech,internet | SG
x.com | X | consumer-tech,internet
threads.net | Threads | consumer-tech,internet
bsky.app | Bluesky | consumer-tech,internet,open-source
joinmastodon.org | Mastodon | consumer-tech,open-source | DE
signal.org | Signal | consumer-tech,cyber,open-source
telegram.org | Telegram | consumer-tech,internet | AE
whatsapp.com | WhatsApp | consumer-tech,internet
peloton.com | Peloton | consumer-tech,health
uber.com | Uber | transport,consumer-tech
lyft.com | Lyft | transport,consumer-tech
doordash.com | DoorDash | food,consumer-tech
instacart.com | Instacart | retail,consumer-tech
airbnb.com | Airbnb | travel,consumer-tech
booking.com | Booking.com | travel | NL
expedia.com | Expedia | travel
hulu.com | Hulu | entertainment,media
max.com | Max | entertainment,media
paramountplus.com | Paramount+ | entertainment,media
peacocktv.com | Peacock | entertainment,media
crunchyroll.com | Crunchyroll | entertainment,media
tidal.com | TIDAL | entertainment,media | NO
deezer.com | Deezer | entertainment,media | FR
soundcloud.com | SoundCloud | entertainment,media | DE
pandora.com | Pandora | entertainment,media
audible.com | Audible | entertainment,media
letterboxd.com | Letterboxd | entertainment,consumer-tech | NZ
fandom.com | Fandom | media,gaming
quora.com | Quora | consumer-tech,internet
tumblr.com | Tumblr | consumer-tech,media
flickr.com | Flickr | consumer-tech,media
dribbble.com | Dribbble | consumer-tech,media
behance.net | Behance | consumer-tech,media
unsplash.com | Unsplash | consumer-tech,media
pexels.com | Pexels | consumer-tech,media | DE
giphy.com | GIPHY | consumer-tech,media
imgur.com | Imgur | consumer-tech,media
wattpad.com | Wattpad | consumer-tech,media | CA
ko-fi.com | Ko-fi | consumer-tech,payments | GB
buymeacoffee.com | Buy Me a Coffee | consumer-tech,payments
kick.com | Kick | media,gaming | AU
rumble.com | Rumble | media | CA
dailymotion.com | Dailymotion | media | FR
nebula.tv | Nebula | media
masterclass.com | MasterClass | education,media
skillshare.com | Skillshare | education,media
edx.org | edX | education
khanacademy.org | Khan Academy | education
brilliant.org | Brilliant | education
codecademy.com | Codecademy | education,developer
datacamp.com | DataCamp | education,developer
pluralsight.com | Pluralsight | education,developer
chegg.com | Chegg | education
quizlet.com | Quizlet | education
blackboard.com | Blackboard (Anthology) | education,saas
moodle.org | Moodle | education,open-source | AU
d2l.com | D2L | education,saas | CA
clever.com | Clever | education,saas
classdojo.com | ClassDojo | education,consumer-tech
remind.com | Remind | education,saas
headspace.com | Headspace | health,consumer-tech
calm.com | Calm | health,consumer-tech
noom.com | Noom | health,consumer-tech
myfitnesspal.com | MyFitnessPal | health,consumer-tech
fitbit.com | Fitbit | health,consumer-tech
garmin.com | Garmin | consumer-tech,transport | CH
withings.com | Withings | health,consumer-tech | FR
zwift.com | Zwift | health,gaming
alltrails.com | AllTrails | consumer-tech,travel
komoot.com | komoot | consumer-tech,travel | DE
transitapp.com | Transit | transport,consumer-tech | CA
moovit.com | Moovit | transport,consumer-tech | IL
lime.co | Lime | transport
bolt.eu | Bolt | transport | EE
grab.com | Grab | transport,payments | SG
careem.com | Careem | transport | AE
blablacar.com | BlaBlaCar | transport | FR
flixbus.com | FlixBus | transport | DE
turo.com | Turo | transport,travel
getaround.com | Getaround | transport
zipcar.com | Zipcar | transport
grubhub.com | Grubhub | food,consumer-tech
deliveroo.com | Deliveroo | food,consumer-tech | GB
justeattakeaway.com | Just Eat Takeaway.com | food,consumer-tech | NL
skipthedishes.com | SkipTheDishes | food,consumer-tech | CA
gopuff.com | Gopuff | retail,consumer-tech
getir.com | Getir | retail,consumer-tech | TR
glovoapp.com | Glovo | food,consumer-tech | ES
wolt.com | Wolt | food,consumer-tech | FI
hellofresh.com | HelloFresh | food,consumer-tech | DE
vrbo.com | Vrbo | travel
hopper.com | Hopper | travel | CA
kayak.com | KAYAK | travel
skyscanner.net | Skyscanner | travel | GB
tripadvisor.com | Tripadvisor | travel
trivago.com | trivago | travel | DE
hostelworld.com | Hostelworld | travel | IE
agoda.com | Agoda | travel | SG
trip.com | Trip.com | travel | SG
marriott.com | Marriott | travel
hilton.com | Hilton | travel
experian.com | Experian | finance | IE
equifax.com | Equifax | finance
transunion.com | TransUnion | finance
creditkarma.com | Credit Karma | finance,consumer-tech
# games
store.steampowered.com | Steam | gaming
xbox.com | Xbox | gaming
nintendo.com | Nintendo | gaming | JP
riotgames.com | Riot Games | gaming
blizzard.com | Blizzard Entertainment | gaming
ea.com | Electronic Arts | gaming
ubisoft.com | Ubisoft | gaming | FR
roblox.com | Roblox | gaming
minecraft.net | Minecraft | gaming
unity.com | Unity | gaming,developer
unrealengine.com | Unreal Engine | gaming,developer
godotengine.org | Godot Engine | gaming,open-source
itch.io | itch.io | gaming
gog.com | GOG | gaming | PL
humblebundle.com | Humble Bundle | gaming
supercell.com | Supercell | gaming | FI
rockstargames.com | Rockstar Games | gaming
bungie.net | Bungie | gaming
squareenix.com | Square Enix | gaming | JP
bethesda.net | Bethesda | gaming
cdprojektred.com | CD Projekt Red | gaming | PL
krafton.com | Krafton | gaming | KR
nexon.com | Nexon | gaming | JP
miHoYo.com | miHoYo | gaming | CN
hoyoverse.com | HoYoverse | gaming | SG
scopely.com | Scopely | gaming
zynga.com | Zynga | gaming
kingdom.com | King | gaming | SE
playtika.com | Playtika | gaming | IL
paradoxplaza.com | Paradox Interactive | gaming | SE
frontier.co.uk | Frontier Developments | gaming | GB
larian.com | Larian Studios | gaming | BE
mojang.com | Mojang | gaming | SE
# security / IT tooling saas (long tail)
cyberark.com | CyberArk | cyber,saas | IL
beyondtrust.com | BeyondTrust | cyber,saas
delinea.com | Delinea | cyber,saas
duo.com | Cisco Duo | cyber,saas
yubico.com | Yubico | cyber | SE
proofpoint.com | Proofpoint | cyber,saas
mimecast.com | Mimecast | cyber,saas | GB
barracuda.com | Barracuda Networks | cyber,saas
zscaler.com | Zscaler | cyber,saas
netskope.com | Netskope | cyber,saas
cloudflare.com | Cloudflare | cloud,cyber
rapid7.com | Rapid7 | cyber,saas
qualys.com | Qualys | cyber,saas
arcticwolf.com | Arctic Wolf | cyber,saas
huntress.com | Huntress | cyber,saas
kandji.io | Kandji | cyber,enterprise,saas
jamf.com | Jamf | enterprise,saas
addigy.com | Addigy | enterprise,saas
mosyle.com | Mosyle | enterprise,saas
ninjaone.com | NinjaOne | enterprise,saas
atera.com | Atera | enterprise,saas | IL
connectwise.com | ConnectWise | enterprise,saas
kaseya.com | Kaseya | enterprise,saas
datto.com | Datto | enterprise,saas
veeam.com | Veeam | enterprise,software | CH
druva.com | Druva | enterprise,saas
backblaze.com | Backblaze | cloud,consumer-tech
carbonite.com | Carbonite | consumer-tech,saas
idrive.com | IDrive | consumer-tech,saas
pcloud.com | pCloud | consumer-tech,saas | CH
sync.com | Sync.com | consumer-tech,saas | CA
mega.io | MEGA | consumer-tech,saas | NZ
icedrive.net | Icedrive | consumer-tech,saas | GB
nordvpn.com | NordVPN | cyber,consumer-tech | PA
expressvpn.com | ExpressVPN | cyber,consumer-tech | VG
mullvad.net | Mullvad | cyber,consumer-tech | SE
protonvpn.com | Proton VPN | cyber,consumer-tech | CH
surfshark.com | Surfshark | cyber,consumer-tech | NL
tunnelbear.com | TunnelBear | cyber,consumer-tech | CA
"""

FEED_PATHS = [
    "/changelog/rss.xml", "/changelog/feed.xml", "/changelog.rss", "/changelog/feed", "/changelog/rss",
    "/releases/feed", "/release-notes/rss.xml", "/updates/rss.xml",
    "/blog/rss.xml", "/blog/feed", "/blog/feed.xml", "/blog/rss",
    "/feed.xml", "/rss.xml", "/feed", "/atom.xml",
]
SUBDOMAIN_FEEDS = [("blog", ["/feed", "/rss.xml", "/feed.xml"]), ("changelog", ["/feed.xml", "/rss.xml"])]
HTML_PAGES = [("/changelog", "changelog", "B"), ("/pricing", "pricing", "C")]
STATUS_JSON_PATHS = ["/api/v2/summary.json", "/summary.json", "/api/v1/summary"]
STATUS_FEED_PATHS = ["/history.rss", "/history.atom", "/feed", "/rss", "/feed.rss", "/index.rss", "/history.xml", "/api/v2/incidents.rss"]
CHALLENGE = re.compile(r"just a moment|cf-chl|attention required|access denied|captcha|enable javascript and cookies|verify you are human|ddos-guard|incapsula|_incapsula", re.I)


class Bucket:
    def __init__(self, rate: float) -> None:
        self.rate, self.tokens, self.t, self.lock = rate, 1.0, time.monotonic(), threading.Lock()

    def take(self) -> None:
        while True:
            with self.lock:
                now = time.monotonic()
                self.tokens = min(1.0 + self.rate, self.tokens + (now - self.t) * self.rate)
                self.t = now
                if self.tokens >= 1:
                    self.tokens -= 1
                    return
                wait = (1 - self.tokens) / self.rate
            time.sleep(wait)


BUCKET = Bucket(RATE)
_local = threading.local()
_dns_cache: dict[str, bool] = {}
_dns_lock = threading.Lock()


def session() -> requests.Session:
    s = getattr(_local, "s", None)
    if s is None:
        s = requests.Session()
        s.headers.update({"User-Agent": UA, "Accept": "application/json, application/rss+xml, application/atom+xml, application/xml, text/xml, text/html;q=0.9, */*;q=0.8", "Accept-Language": "en"})
        _local.s = s
    return s


def resolves(host: str) -> bool:
    with _dns_lock:
        if host in _dns_cache:
            return _dns_cache[host]
    try:
        socket.getaddrinfo(host, 443, proto=socket.IPPROTO_TCP)
        ok = True
    except OSError:
        ok = False
    with _dns_lock:
        _dns_cache[host] = ok
    return ok


def get(url: str) -> dict | None:
    host = urlparse(url).hostname or ""
    if not resolves(host):
        return None
    BUCKET.take()
    try:
        r = session().get(url, timeout=TIMEOUT, allow_redirects=True, stream=True)
        body = b""
        for chunk in r.iter_content(65536):
            body += chunk
            if len(body) > MAX_BODY:
                break
        r.close()
        text = body.decode(r.encoding or "utf-8", errors="replace") if body else ""
        return {"status": r.status_code, "ct": (r.headers.get("content-type") or "").lower(), "final": r.url, "text": text, "len": len(body)}
    except requests.RequestException:
        return None


def json_shape(text: str) -> tuple[str, dict] | None:
    t = text.lstrip(" \n\r\t")
    if not t.startswith("{"):
        return None
    try:
        j = json.loads(t)
    except ValueError:
        return None
    if not isinstance(j, dict):
        return None
    page = j.get("page")
    if isinstance(page, dict) and isinstance(j.get("status"), dict) and ("components" in j or "incidents" in j) and "id" in page:
        return "statuspage", j
    if isinstance(j.get("subpages"), list) and any(isinstance(p, dict) and "summary" in p for p in j["subpages"]):
        return "incidentio", j
    if isinstance(page, dict) and ("activeIncidents" in j or "activeMaintenances" in j or isinstance(page.get("status"), str)):
        return "instatus", j
    if "ongoing_incidents" in j or "page_title" in j:
        return "incidentio", j
    if isinstance(j.get("result"), dict) and "status_overall" in j["result"]:
        return "statusio", j
    return None


def feed_shape(text: str, ct: str) -> dict | None:
    head = text[:4000].lstrip(" \n\r\t")
    if "text/html" in ct and not head.startswith("<?xml") and not re.match(r"<(rss|feed|rdf:RDF)\b", head):
        return None
    if not re.search(r"<(rss|feed|rdf:RDF)[\s>]", head):
        return None
    items = len(re.findall(r"<(item|entry)[\s>]", text))
    if items == 0:
        return None
    title = re.search(r"<title[^>]*>(?:<!\[CDATA\[)?(.*?)(?:\]\]>)?</title>", text, re.S)
    dates = re.findall(r"<(?:pubDate|updated|published|dc:date)[^>]*>([^<]+)<", text)
    return {"items": items, "title": (title.group(1).strip()[:80] if title else ""), "latest": dates[0].strip() if dates else "", "hash": hashlib.sha1(re.sub(r"\s+", " ", text[:20000]).encode()).hexdigest()[:12]}


def html_text_len(text: str) -> int:
    t = re.sub(r"<(script|style|noscript|svg)[^>]*>.*?</\1>", " ", text, flags=re.S | re.I)
    t = re.sub(r"<[^>]+>", " ", t)
    t = re.sub(r"\s+", " ", t)
    return len(t)


def parse_candidates() -> list[dict]:
    out, seen = [], set()
    for ln in CANDIDATES.strip().splitlines():
        ln = ln.strip()
        if not ln or ln.startswith("#"):
            continue
        parts = [p.strip() for p in ln.split("|")]
        domain = parts[0].lower()
        if domain in seen:
            continue
        seen.add(domain)
        out.append({"domain": domain, "name": parts[1], "categories": [c.strip() for c in parts[2].split(",")], "country": parts[3] if len(parts) > 3 else "US"})
    return out


def load_registry() -> tuple[dict[str, str], set[str], set[str], set[str]]:
    """id→domain, all sensor URLs, hosts of existing status sensors, all ids."""
    id_domain: dict[str, str] = {}
    urls: set[str] = set()
    status_hosts: set[str] = set()
    ids: set[str] = set()
    for f in ["config/sources.yaml"] + sorted(glob.glob("config/sources.d/*.yaml")):
        if f.endswith("48-saas-status-changelogs.yaml"):
            continue
        d = yaml.safe_load(open(f)) or {}
        for s in d.get("sources", []):
            ids.add(s["id"])
            if s.get("domain"):
                id_domain[s["id"]] = s["domain"].lower()
            for sen in s.get("sensors", []) or []:
                u = sen.get("url", "")
                urls.add(u.rstrip("/"))
                if sen.get("type") == "STATUSPAGE" or sen.get("connector") in ("statuspage", "statusjson") or "status" in (sen.get("name") or ""):
                    h = urlparse(u).hostname
                    if h:
                        status_hosts.add(h)
    return id_domain, urls, status_hosts, ids


def apex(host: str) -> str:
    parts = host.split(".")
    if len(parts) >= 3 and parts[-2] in ("co", "com", "org", "gov", "ac", "net") and len(parts[-1]) == 2:
        return ".".join(parts[-3:])
    return ".".join(parts[-2:])


def probe_domain(c: dict, existing_urls: set[str], status_hosts: set[str]) -> dict:
    d = c["domain"]
    hits: list[dict] = []
    notes: list[str] = []
    probed_hosts: set[str] = set()

    def try_status_host(host: str, label: str) -> bool:
        if host in probed_hosts:
            return False
        if host in status_hosts:
            notes.append(f"{host}: already covered")
            return True
        probed_hosts.add(host)
        for p in STATUS_JSON_PATHS:
            url = f"https://{host}{p}"
            if url in existing_urls:
                return True
            if not resolves(host):
                return False
            r = get(url)
            if not r:
                continue
            if r["status"] == 200:
                sh = json_shape(r["text"])
                if sh:
                    flavor, j = sh
                    title = (j.get("page") or {}).get("name") if isinstance(j.get("page"), dict) else j.get("page_title", "")
                    hits.append({"kind": "status", "flavor": flavor, "url": url, "final": r["final"], "title": title or "", "via": label})
                    return True
        # HTML root → provider sniffing
        r = get(f"https://{host}/")
        if r and r["status"] == 200:
            body = r["text"]
            m = re.search(r"statusio_page_id['\"]?\s*[:=]\s*['\"]([0-9a-f]{20,})", body) or re.search(r"api\.status\.io/1\.0/status/([0-9a-f]{20,})", body)
            if m:
                url = f"https://api.status.io/1.0/status/{m.group(1)}"
                rr = get(url)
                if rr and rr["status"] == 200 and json_shape(rr["text"]):
                    hits.append({"kind": "status", "flavor": "statusio", "url": url, "final": rr["final"], "title": "", "via": label})
                    return True
            prov = "hund" if "hund.io" in body else "sorry" if "sorryapp" in body else "firehydrant" if "firehydrant" in body else "betterstack" if ("betteruptime" in body or "betterstack" in body) else "freshstatus" if "freshstatus" in body else "statuspage-html" if "statuspage.io" in body else "custom"
            if prov == "betterstack":
                notes.append(f"{host}: Better Stack status page (no JSON API)")
                return False
            for p in STATUS_FEED_PATHS:
                url = f"https://{host}{p}"
                if url in existing_urls:
                    return True
                rr = get(url)
                if rr and rr["status"] == 200:
                    fs = feed_shape(rr["text"], rr["ct"])
                    if fs:
                        hits.append({"kind": "status-feed", "flavor": prov, "url": url, "final": rr["final"], "title": fs["title"], "items": fs["items"], "latest": fs["latest"], "via": label})
                        return True
            notes.append(f"{host}: {prov} status page, no machine-readable endpoint")
        elif r and r["status"] in (403, 429, 503):
            notes.append(f"{host}: http {r['status']}")
        return False

    # 1. status.<domain> (and where its root redirects)
    sh = f"status.{apex(d)}"
    if sh not in status_hosts:
        found = False
        if resolves(sh):
            root = get(f"https://{sh}/")
            if root:
                fh = urlparse(root["final"]).hostname or sh
                found = try_status_host(sh, "status host")
                if not found and fh != sh:
                    found = try_status_host(fh, f"redirect from {sh}")
            else:
                notes.append(f"{sh}: unreachable")
        # 2. <domain>/status redirecting to a status host
        if not found:
            r = get(f"https://{d}/status")
            if r and r["status"] == 200:
                fh = urlparse(r["final"]).hostname or d
                fp = urlparse(r["final"]).path
                if fh not in (d, f"www.{d}") and fh not in probed_hosts:
                    try_status_host(fh, f"redirect from {d}/status")
                elif fp.rstrip("/").endswith("/status") and json_shape(r["text"]) is None and "text/html" in r["ct"]:
                    if "statuspage.io" in r["text"] or "instatus" in r["text"]:
                        m = re.search(r"https://([a-z0-9.-]+)/api/v2/", r["text"])
                        if m:
                            try_status_host(m.group(1), f"embedded in {d}/status")
    else:
        notes.append(f"{sh}: already covered")

    # 3. feeds on the main domain
    feed_hits = 0
    seen_final: set[str] = set()
    seen_hash: set[str] = set()
    dead = 0
    for p in FEED_PATHS:
        if feed_hits >= 2 or dead >= 3:
            break
        url = f"https://{d}{p}"
        if url in existing_urls:
            continue
        r = get(url)
        if not r:
            dead += 1
            continue
        if r["status"] in (403, 429) and p == FEED_PATHS[0]:
            notes.append(f"{d}: http {r['status']} (blocked)")
            dead = 3
            break
        if r["status"] != 200:
            continue
        fs = feed_shape(r["text"], r["ct"])
        if not fs:
            continue
        fk = r["final"].rstrip("/")
        if fk in seen_final or fs["hash"] in seen_hash or fk in existing_urls:
            continue
        seen_final.add(fk)
        seen_hash.add(fs["hash"])
        hits.append({"kind": "feed", "path": p, "url": url, "final": r["final"], "title": fs["title"], "items": fs["items"], "latest": fs["latest"]})
        feed_hits += 1
    # 4. blog./changelog. subdomains
    for sub, paths in SUBDOMAIN_FEEDS:
        host = f"{sub}.{apex(d)}"
        if not resolves(host):
            continue
        for p in paths:
            url = f"https://{host}{p}"
            if url in existing_urls:
                break
            r = get(url)
            if not r or r["status"] != 200:
                continue
            fs = feed_shape(r["text"], r["ct"])
            if not fs:
                continue
            fk = r["final"].rstrip("/")
            if fk in seen_final or fs["hash"] in seen_hash or fk in existing_urls:
                break
            seen_final.add(fk)
            seen_hash.add(fs["hash"])
            hits.append({"kind": "feed", "path": f"{sub}:{p}", "url": url, "final": r["final"], "title": fs["title"], "items": fs["items"], "latest": fs["latest"]})
            break
    # 5. server-rendered HTML pages
    for p, label, tier in HTML_PAGES:
        url = f"https://{d}{p}"
        if url in existing_urls or f"https://www.{d}{p}" in existing_urls:
            continue
        r = get(url)
        if not r or r["status"] != 200 or "text/html" not in r["ct"]:
            continue
        fpath = urlparse(r["final"]).path.rstrip("/")
        if fpath in ("", "/") or CHALLENGE.search(r["text"][:6000]):
            continue
        n = html_text_len(r["text"])
        if n >= 2500:
            hits.append({"kind": "html", "label": label, "tier": tier, "url": url, "final": r["final"], "textlen": n})
    return {"domain": d, "hits": hits, "notes": notes}


def cmd_probe(args: argparse.Namespace) -> None:
    id_domain, urls, status_hosts, _ids = load_registry()
    cands = parse_candidates()
    if args.only:
        only = set(args.only.split(","))
        cands = [c for c in cands if c["domain"] in only]
    done: set[str] = set()
    if os.path.exists(args.out) and not args.fresh:
        for ln in open(args.out):
            try:
                done.add(json.loads(ln)["domain"])
            except ValueError:
                pass
    todo = [c for c in cands if c["domain"] not in done]
    print(f"{len(cands)} candidates, {len(done)} already probed, {len(todo)} to go · ≤ {RATE} req/s", file=sys.stderr)
    lock = threading.Lock()
    t0 = time.time()
    with open(args.out, "a") as out, ThreadPoolExecutor(max_workers=args.workers) as ex:
        futs = {ex.submit(probe_domain, c, urls, status_hosts): c for c in todo}
        for i, fut in enumerate(as_completed(futs), 1):
            c = futs[fut]
            try:
                res = fut.result()
            except Exception as e:  # noqa: BLE001
                res = {"domain": c["domain"], "hits": [], "notes": [f"probe error: {e}"]}
            res.update({"name": c["name"], "categories": c["categories"], "country": c["country"]})
            with lock:
                out.write(json.dumps(res) + "\n")
                out.flush()
            kinds = ",".join(sorted({h["kind"] + (":" + h["flavor"] if h.get("flavor") else "") for h in res["hits"]})) or "-"
            print(f"[{i}/{len(todo)} {time.time() - t0:5.0f}s] {c['domain']:28s} {kinds}", file=sys.stderr)


def slug(s: str) -> str:
    s = s.lower().replace("&", " and ")
    s = re.sub(r"\(.*?\)", "", s)
    s = re.sub(r"[^a-z0-9]+", "-", s).strip("-")
    return s


# Same organization declared earlier under another domain → extend instead of redeclaring.
EXTEND_OVERRIDE = {"deliveroo.com": "deliveroo", "greenhouse.io": "greenhouse", "notion.so": "notion", "peloton.com": "peloton",
                   "timescale.com": "timescale", "sparkpost.com": "bird", "unrealengine.com": "epic-games"}
# Different organization whose natural slug is taken (bolt = Bolt EE ride-hailing, census = US Census Bureau).
NEW_ID_OVERRIDE = {"bolt.com": "bolt-payments", "getcensus.com": "getcensus"}


def cmd_emit(args: argparse.Namespace) -> None:
    id_domain, urls, status_hosts, ids = load_registry()
    domain_id = dict(EXTEND_OVERRIDE)
    for i, dom in id_domain.items():
        domain_id.setdefault(dom, i)
        domain_id.setdefault(apex(dom), i)
    rows = [json.loads(ln) for ln in open(args.inp) if ln.strip()]
    rows.sort(key=lambda r: r["domain"])
    used_urls: set[str] = set()
    new_ids: set[str] = set()
    entries: dict[str, tuple[list[str], list[str]]] = {}
    print("# config/sources.d/48-saas-status-changelogs.yaml — 2026-09-11 — the long tail of SaaS / B2B / consumer-internet")
    print("# services: machine-readable status pages (Atlassian Statuspage summary API, Instatus summary.json, incident.io")
    print("# /api/v1/summary, Status.io, RSS history feeds of Hund/Sorry/FireHydrant pages), changelog / release-note / blog")
    print("# feeds and server-rendered changelog / pricing pages. Generated by scripts/gen-status-probe.py (probe of ~600")
    print("# candidate domains at ≤ 4 req/s, 2026-09-11), curated by hand, then pruned with the live validator: every sensor")
    print("# below was fetched and parsed by apps/engine/src/validate.ts. Organizations already declared in files 10–46 are")
    print("# extended (`extend: true`); the others are declared here with domain, categories, tier and country.")
    print("sources:")
    for r in rows:
        if not r["hits"]:
            continue
        d = r["domain"]
        existing = domain_id.get(d) or domain_id.get(apex(d)) or domain_id.get(f"www.{d}")
        sensors: list[str] = []
        has_status = False
        for h in r["hits"]:
            u = h["url"].rstrip("/")
            if u in urls or u in used_urls or h["final"].rstrip("/") in urls:
                continue
            used_urls.add(u)
            if h["kind"] == "status":
                has_status = True
                if h["flavor"] == "statuspage":
                    sensors.append(f'      - {{ name: status, url: "{h["url"]}", type: STATUSPAGE, connector: statuspage, tier: S }}')
                else:
                    label = {"instatus": "instatus", "incidentio": "incident.io", "statusio": "status.io"}[h["flavor"]]
                    sensors.append(f'      - {{ name: status ({label}), url: "{h["url"]}", type: STATUSPAGE, connector: statusjson, tier: S }}')
            elif h["kind"] == "status-feed":
                sensors.append(f'      - {{ name: status history feed, url: "{h["url"]}", type: RSS, connector: rss, tier: A }}')
            elif h["kind"] == "feed":
                p = h.get("path", "")
                name = "changelog feed" if "changelog" in p or "release" in p or "updates" in p or "whats-new" in p or "product-updates" in p else "blog feed" if "blog" in p else "news feed" if "news" in p else "feed"
                cap = ", config: { maxItems: 60 }" if (h.get("items") or 0) > 60 else ""
                sensors.append(f'      - {{ name: {name}, url: "{h["url"]}", type: {"ATOM" if p.endswith("atom.xml") else "RSS"}, connector: rss, tier: B{cap} }}')
            elif h["kind"] == "html":
                sensors.append(f'      - {{ name: {h["label"]}, url: "{h["url"]}", type: HTML, connector: http, tier: {h["tier"]} }}')
        if not sensors:
            continue
        # dedupe sensor names within the source (feed vs feed)
        seen_names: dict[str, int] = {}
        fixed = []
        for s in sensors:
            m = re.match(r"      - \{ name: ([^,]+),", s)
            nm = m.group(1)
            if nm in seen_names:
                seen_names[nm] += 1
                s = s.replace(f"name: {nm},", f"name: {nm} {seen_names[nm]},", 1)
            else:
                seen_names[nm] = 1
            fixed.append(s)
        if existing:
            head = [f"  - id: {existing}", "    extend: true"]
            if r.get("country"):
                head.append(f"    country: {r['country']}")
            head.append("    sensors:")
            key = existing
        else:
            sid = NEW_ID_OVERRIDE.get(d) or slug(r["name"]) or slug(d.split(".")[0])
            if sid in ids or sid in new_ids:
                sid = slug(d.replace(".", "-"))
            if sid in ids or sid in new_ids:
                sid = f"{sid}-saas"
            new_ids.add(sid)
            cats = ", ".join(r["categories"])
            head = [f"  - id: {sid}", f"    name: {r['name']}", f"    domain: {d}", f"    categories: [{cats}]", f"    tier: {'S' if has_status else 'B'}",
                    f"    country: {r['country']}", "    discover: { rss: true, status: true }", "    sensors:"]
            key = sid
        if key in entries:  # same organization reached through two domains → merge the sensor lists
            entries[key][1].extend(fixed)
        else:
            entries[key] = (head, list(fixed))
    for head, sens in entries.values():
        for ln in head:
            print(ln)
        for s in sens:
            print(s)
    # notes for the header
    blocked = [n for r in rows for n in r["notes"] if "already covered" not in n and "unreachable" not in n]
    print("# ── probe notes (not sensors) ──", file=sys.stderr)
    for n in sorted(set(blocked)):
        print(f"#   {n}", file=sys.stderr)


def main() -> None:
    ap = argparse.ArgumentParser()
    sub = ap.add_subparsers(dest="cmd", required=True)
    p = sub.add_parser("probe")
    p.add_argument("--out", default="/tmp/probe48.jsonl")
    p.add_argument("--only")
    p.add_argument("--fresh", action="store_true")
    p.add_argument("--workers", type=int, default=8)
    p.set_defaults(fn=cmd_probe)
    e = sub.add_parser("emit")
    e.add_argument("--in", dest="inp", default="/tmp/probe48.jsonl")
    e.set_defaults(fn=cmd_emit)
    a = ap.parse_args()
    a.fn(a)


if __name__ == "__main__":
    main()