TypeScript 55.4%
Python 43.2%
SQL 1.2%
1#!/usr/bin/env python32"""Probe the long tail of SaaS / B2B / consumer-internet services for machine-readable status pages,3changelog / release-note feeds and server-rendered changelog / pricing pages, then emit a registry4fragment to curate (config/sources.d/48-saas-status-changelogs.yaml).56Usage:7 python3 scripts/gen-status-probe.py probe [--out /tmp/probe48.jsonl] [--only domain,domain]8 python3 scripts/gen-status-probe.py emit [--in /tmp/probe48.jsonl] > config/sources.d/48-saas-status-changelogs.yaml9Then validate with apps/engine/src/validate.ts and prune with scripts/prune-fragment.py.1011Politeness: one global token bucket (≤ 4 requests/s), 15 s timeout, ≤ 1.5 MB per body, UA12"WebSensor registry probe (contact@websensor.io)". Hosts that do not resolve cost no request.13Duplicates: URLs already present anywhere in config/ are never emitted; status hosts already covered by14an existing STATUSPAGE sensor are not probed again; ids that exist in an earlier file are `extend: true`.15"""16from __future__ import annotations1718import argparse19import glob20import hashlib21import json22import os23import re24import socket25import sys26import threading27import time28from concurrent.futures import ThreadPoolExecutor, as_completed29from urllib.parse import urlparse3031import requests32import yaml3334UA = "WebSensor registry probe (contact@websensor.io)"35TIMEOUT = 1536MAX_BODY = 1_500_00037RATE = 4.0 # requests per second, global3839# ───────────────────────── candidates: domain | Name | categories | country (default US) ─────────────────────────40CANDIDATES = """41# collaboration & productivity42slack.com | Slack | enterprise,saas43zoom.us | Zoom | enterprise,saas44notion.so | Notion | enterprise,saas45figma.com | Figma | enterprise,saas,developer46miro.com | Miro | enterprise,saas47loom.com | Loom | enterprise,saas48canva.com | Canva | consumer-tech,saas | AU49airtable.com | Airtable | enterprise,saas50asana.com | Asana | enterprise,saas51monday.com | monday.com | enterprise,saas | IL52clickup.com | ClickUp | enterprise,saas53linear.app | Linear | developer,saas54basecamp.com | Basecamp | enterprise,saas55trello.com | Trello | enterprise,saas56coda.io | Coda | enterprise,saas57dropbox.com | Dropbox | enterprise,consumer-tech58box.com | Box | enterprise,saas59zapier.com | Zapier | enterprise,saas60make.com | Make | enterprise,saas | CZ61n8n.io | n8n | developer,open-source,saas | DE62calendly.com | Calendly | enterprise,saas63cal.com | Cal.com | developer,open-source,saas64typeform.com | Typeform | enterprise,saas | ES65surveymonkey.com | SurveyMonkey | enterprise,saas66qualtrics.com | Qualtrics | enterprise,saas67evernote.com | Evernote | consumer-tech,saas68todoist.com | Todoist | consumer-tech,saas69ticktick.com | TickTick | consumer-tech,saas70obsidian.md | Obsidian | consumer-tech,developer71craft.do | Craft | consumer-tech,saas72whimsical.com | Whimsical | enterprise,saas73lucid.co | Lucid Software | enterprise,saas74mural.co | Mural | enterprise,saas75pitch.com | Pitch | enterprise,saas | DE76gamma.app | Gamma | enterprise,saas,ai77tldraw.com | tldraw | developer,open-source78excalidraw.com | Excalidraw | developer,open-source79docusign.com | Docusign | enterprise,saas,legal80hellosign.com | Dropbox Sign | enterprise,saas81pandadoc.com | PandaDoc | enterprise,saas82ironcladapp.com | Ironclad | enterprise,saas,legal831password.com | 1Password | cyber,saas | CA84lastpass.com | LastPass | cyber,saas85bitwarden.com | Bitwarden | cyber,open-source,saas86dashlane.com | Dashlane | cyber,saas | FR87keeper.io | Keeper Security | cyber,saas88grammarly.com | Grammarly | consumer-tech,saas,ai89otter.ai | Otter.ai | enterprise,saas,ai90fireflies.ai | Fireflies.ai | enterprise,saas,ai91gong.io | Gong | enterprise,saas | IL92clari.com | Clari | enterprise,saas93outreach.io | Outreach | enterprise,saas94salesloft.com | Salesloft | enterprise,saas95apollo.io | Apollo.io | enterprise,saas96zoominfo.com | ZoomInfo | enterprise,saas97lusha.com | Lusha | enterprise,saas | IL98clearbit.com | Clearbit | enterprise,saas99drift.com | Drift | enterprise,saas100front.com | Front | enterprise,saas101helpscout.com | Help Scout | enterprise,saas102gorgias.com | Gorgias | enterprise,saas,commerce103kustomer.com | Kustomer | enterprise,saas104crisp.chat | Crisp | enterprise,saas | FR105tawk.to | tawk.to | enterprise,saas106livechat.com | LiveChat | enterprise,saas | PL107text.com | Text (LiveChat) | enterprise,saas | PL108tidio.com | Tidio | enterprise,saas | PL109olark.com | Olark | enterprise,saas110dialpad.com | Dialpad | enterprise,telecom,saas111aircall.io | Aircall | enterprise,telecom,saas | FR112ringcentral.com | RingCentral | enterprise,telecom,saas1138x8.com | 8x8 | enterprise,telecom,saas114webex.com | Webex | enterprise,telecom,saas115goto.com | GoTo | enterprise,telecom,saas116whereby.com | Whereby | enterprise,saas | NO117teamviewer.com | TeamViewer | enterprise,saas | DE118anydesk.com | AnyDesk | enterprise,saas | DE119splashtop.com | Splashtop | enterprise,saas120smartsheet.com | Smartsheet | enterprise,saas121wrike.com | Wrike | enterprise,saas122teamwork.com | Teamwork.com | enterprise,saas | IE123productboard.com | Productboard | enterprise,saas | CZ124aha.io | Aha! | enterprise,saas125canny.io | Canny | enterprise,saas126featurebase.app | Featurebase | enterprise,saas127shortcut.com | Shortcut | developer,saas128height.app | Height | developer,saas129plane.so | Plane | developer,open-source,saas130fibery.io | Fibery | enterprise,saas | CY131lattice.com | Lattice | enterprise,saas132cultureamp.com | Culture Amp | enterprise,saas | AU13315five.com | 15Five | enterprise,saas134leapsome.com | Leapsome | enterprise,saas | DE135workday.com | Workday | enterprise,saas136bamboohr.com | BambooHR | enterprise,saas137gusto.com | Gusto | enterprise,payments,saas138rippling.com | Rippling | enterprise,saas139deel.com | Deel | enterprise,labour,saas140remote.com | Remote | enterprise,labour,saas141justworks.com | Justworks | enterprise,saas142trinet.com | TriNet | enterprise,saas143paychex.com | Paychex | enterprise,payments,saas144paylocity.com | Paylocity | enterprise,saas145paycom.com | Paycom | enterprise,saas146personio.de | Personio | enterprise,saas | DE147hibob.com | HiBob | enterprise,saas | IL148greenhouse.io | Greenhouse | enterprise,labour,saas149lever.co | Lever | enterprise,labour,saas150ashbyhq.com | Ashby | enterprise,labour,saas151workable.com | Workable | enterprise,labour,saas152smartrecruiters.com | SmartRecruiters | enterprise,labour,saas153icims.com | iCIMS | enterprise,labour,saas154checkr.com | Checkr | enterprise,saas155vanta.com | Vanta | cyber,enterprise,saas156drata.com | Drata | cyber,enterprise,saas157secureframe.com | Secureframe | cyber,enterprise,saas158dropboxforum.com | Dropbox Forum | enterprise159intercom.com | Intercom | enterprise,saas160zendesk.com | Zendesk | enterprise,saas161freshworks.com | Freshworks | enterprise,saas162hubspot.com | HubSpot | enterprise,saas163pipedrive.com | Pipedrive | enterprise,saas | EE164zoho.com | Zoho | enterprise,saas | IN165mailchimp.com | Mailchimp | enterprise,saas166klaviyo.com | Klaviyo | enterprise,commerce,saas167braze.com | Braze | enterprise,saas168iterable.com | Iterable | enterprise,saas169customer.io | Customer.io | enterprise,saas170segment.com | Twilio Segment | developer,saas171amplitude.com | Amplitude | enterprise,saas172mixpanel.com | Mixpanel | enterprise,saas173heap.io | Heap | enterprise,saas174hotjar.com | Hotjar | enterprise,saas | MT175fullstory.com | Fullstory | enterprise,saas176launchdarkly.com | LaunchDarkly | developer,saas177optimizely.com | Optimizely | enterprise,saas178split.io | Split | developer,saas179statsig.com | Statsig | developer,saas180posthog.com | PostHog | developer,open-source,saas181rollbar.com | Rollbar | developer,saas182bugsnag.com | Bugsnag | developer,saas183opsgenie.com | Opsgenie | developer,saas184firehydrant.com | FireHydrant | developer,saas185incident.io | incident.io | developer,saas | GB186statuspage.io | Atlassian Statuspage | developer,saas187# developer platforms & APIs188clerk.com | Clerk | developer,saas189workos.com | WorkOS | developer,saas190stytch.com | Stytch | developer,saas191descope.com | Descope | developer,saas192frontegg.com | Frontegg | developer,saas | IL193supertokens.com | SuperTokens | developer,open-source194postmarkapp.com | Postmark | developer,saas195mailgun.com | Mailgun | developer,saas196mailtrap.io | Mailtrap | developer,saas197mailjet.com | Mailjet | developer,saas | FR198sparkpost.com | SparkPost | developer,saas199courier.com | Courier | developer,saas200knock.app | Knock | developer,saas201onesignal.com | OneSignal | developer,saas202novu.co | Novu | developer,open-source,saas203algolia.com | Algolia | developer,saas | FR204typesense.org | Typesense | developer,open-source205meilisearch.com | Meilisearch | developer,open-source,saas | FR206cloudinary.com | Cloudinary | developer,saas | IL207imgix.com | imgix | developer,saas208uploadcare.com | Uploadcare | developer,saas209filestack.com | Filestack | developer,saas210mux.com | Mux | developer,saas211vimeo.com | Vimeo | media,saas212wistia.com | Wistia | enterprise,media,saas213bitmovin.com | Bitmovin | developer,media,saas | AT214agora.io | Agora | developer,telecom,saas215daily.co | Daily | developer,telecom,saas216livekit.io | LiveKit | developer,open-source,telecom217100ms.live | 100ms | developer,telecom,saas | IN218ably.com | Ably | developer,saas | GB219pusher.com | Pusher | developer,saas | GB220pubnub.com | PubNub | developer,saas221getstream.io | Stream | developer,saas222sendbird.com | Sendbird | developer,saas223vonage.com | Vonage | developer,telecom,saas224bird.com | Bird | developer,telecom,saas | NL225plivo.com | Plivo | developer,telecom,saas226telnyx.com | Telnyx | developer,telecom,saas227sinch.com | Sinch | developer,telecom,saas | SE228bandwidth.com | Bandwidth | developer,telecom,saas229infobip.com | Infobip | developer,telecom,saas | HR230postman.com | Postman | developer,saas231readme.com | ReadMe | developer,saas232mintlify.com | Mintlify | developer,saas233gitbook.com | GitBook | developer,saas234swagger.io | Swagger | developer,open-source235stoplight.io | Stoplight | developer,saas236konghq.com | Kong | developer,open-source,saas237tyk.io | Tyk | developer,open-source,saas | GB238zuplo.com | Zuplo | developer,saas239hookdeck.com | Hookdeck | developer,saas | CA240svix.com | Svix | developer,saas241ngrok.com | ngrok | developer,saas242goteleport.com | Teleport | developer,cyber,saas243doppler.com | Doppler | developer,cyber,saas244infisical.com | Infisical | developer,open-source,cyber245pulumi.com | Pulumi | developer,saas246spacelift.io | Spacelift | developer,saas247env0.com | env0 | developer,saas | IL248scalr.com | Scalr | developer,saas249circleci.com | CircleCI | developer,saas250buildkite.com | Buildkite | developer,saas | AU251travis-ci.com | Travis CI | developer,saas252semaphoreci.com | Semaphore CI | developer,saas253codemagic.io | Codemagic | developer,saas | EE254bitrise.io | Bitrise | developer,saas | HU255expo.dev | Expo | developer,open-source,saas256harness.io | Harness | developer,saas257octopus.com | Octopus Deploy | developer,saas | AU258codecov.io | Codecov | developer,saas259coveralls.io | Coveralls | developer,saas260sonarsource.com | SonarSource | developer,saas | CH261snyk.io | Snyk | cyber,developer,saas262mend.io | Mend.io | cyber,developer,saas | IL263jfrog.com | JFrog | developer,saas | IL264sonatype.com | Sonatype | developer,cyber,saas265cloudsmith.com | Cloudsmith | developer,saas | GB266replit.com | Replit | developer,saas,ai267codesandbox.io | CodeSandbox | developer,saas268stackblitz.com | StackBlitz | developer,saas269gitpod.io | Gitpod | developer,saas | CH270coder.com | Coder | developer,open-source,saas271cursor.com | Cursor | developer,ai,saas272windsurf.com | Windsurf | developer,ai,saas273sourcegraph.com | Sourcegraph | developer,saas274warp.dev | Warp | developer,saas275raycast.com | Raycast | developer,consumer-tech276jetbrains.com | JetBrains | developer,software | CZ277orbstack.dev | OrbStack | developer,software278localstack.cloud | LocalStack | developer,saas | CH279browserstack.com | BrowserStack | developer,saas | IN280saucelabs.com | Sauce Labs | developer,saas281lambdatest.com | LambdaTest | developer,saas | IN282cypress.io | Cypress | developer,open-source,saas283checklyhq.com | Checkly | developer,saas | DE284honeycomb.io | Honeycomb | developer,saas285chronosphere.io | Chronosphere | developer,saas286betterstack.com | Better Stack | developer,saas | CZ287uptimerobot.com | UptimeRobot | developer,saas | MT288pingdom.com | Pingdom | developer,saas289statuscake.com | StatusCake | developer,saas | GB290cronitor.io | Cronitor | developer,saas291healthchecks.io | Healthchecks.io | developer,open-source,saas292squadcast.com | Squadcast | developer,saas293rootly.com | Rootly | developer,saas294splunk.com | Splunk | enterprise,cyber,saas295sumologic.com | Sumo Logic | enterprise,developer,saas296logz.io | Logz.io | developer,saas | IL297axiom.co | Axiom | developer,saas298highlight.io | Highlight.io | developer,open-source,saas299logrocket.com | LogRocket | developer,saas300smartlook.com | Smartlook | enterprise,saas | CZ301mouseflow.com | Mouseflow | enterprise,saas | DK302contentsquare.com | Contentsquare | enterprise,saas | FR303pendo.io | Pendo | enterprise,saas304appcues.com | Appcues | enterprise,saas305userpilot.com | Userpilot | enterprise,saas306chameleon.io | Chameleon | enterprise,saas307walkme.com | WalkMe | enterprise,saas | IL308whatfix.com | Whatfix | enterprise,saas | IN309contentful.com | Contentful | developer,saas | DE310sanity.io | Sanity | developer,saas | NO311strapi.io | Strapi | developer,open-source,saas | FR312storyblok.com | Storyblok | developer,saas | AT313hygraph.com | Hygraph | developer,saas | DE314prismic.io | Prismic | developer,saas | FR315directus.io | Directus | developer,open-source,saas316payloadcms.com | Payload | developer,open-source317builder.io | Builder.io | developer,saas318plasmic.app | Plasmic | developer,saas319webflow.com | Webflow | developer,saas320framer.com | Framer | developer,saas | NL321squarespace.com | Squarespace | commerce,saas322wix.com | Wix | commerce,saas | IL323wordpress.com | WordPress.com | media,saas324ghost.org | Ghost | media,open-source,saas325substack.com | Substack | media,saas326medium.com | Medium | media,saas327beehiiv.com | beehiiv | media,saas328kit.com | Kit (ConvertKit) | media,saas329buttondown.com | Buttondown | media,saas330mailerlite.com | MailerLite | enterprise,saas | LT331activecampaign.com | ActiveCampaign | enterprise,saas332constantcontact.com | Constant Contact | enterprise,saas333campaignmonitor.com | Campaign Monitor | enterprise,saas | AU334brevo.com | Brevo | enterprise,saas | FR335getresponse.com | GetResponse | enterprise,saas | PL336omnisend.com | Omnisend | enterprise,commerce,saas | LT337attentive.com | Attentive | enterprise,commerce,saas338postscript.io | Postscript | enterprise,commerce,saas339yotpo.com | Yotpo | enterprise,commerce,saas | IL340rechargepayments.com | Recharge | commerce,payments,saas341sproutsocial.com | Sprout Social | enterprise,saas342hootsuite.com | Hootsuite | enterprise,saas | CA343buffer.com | Buffer | enterprise,saas344later.com | Later | enterprise,saas | CA345sprinklr.com | Sprinklr | enterprise,saas346brandwatch.com | Brandwatch | enterprise,saas | GB347meltwater.com | Meltwater | enterprise,saas | NO348semrush.com | Semrush | enterprise,saas349ahrefs.com | Ahrefs | enterprise,saas | SG350moz.com | Moz | enterprise,saas351similarweb.com | Similarweb | enterprise,saas | IL352crazyegg.com | Crazy Egg | enterprise,saas353vwo.com | VWO | enterprise,saas | IN354abtasty.com | AB Tasty | enterprise,saas | FR355unbounce.com | Unbounce | enterprise,saas | CA356instapage.com | Instapage | enterprise,saas357carrd.co | Carrd | consumer-tech,saas358linktr.ee | Linktree | consumer-tech,saas | AU359bitly.com | Bitly | enterprise,saas360dub.co | Dub | developer,saas361short.io | Short.io | enterprise,saas362savvycal.com | SavvyCal | enterprise,saas363doodle.com | Doodle | consumer-tech,saas | CH364# data, analytics & internal tools365getdbt.com | dbt Labs | developer,saas366fivetran.com | Fivetran | developer,saas367airbyte.com | Airbyte | developer,open-source,saas368stitchdata.com | Stitch | developer,saas369hightouch.com | Hightouch | developer,saas370getcensus.com | Census | developer,saas371looker.com | Looker | enterprise,saas372tableau.com | Tableau | enterprise,saas373metabase.com | Metabase | developer,open-source,saas374mode.com | Mode | enterprise,saas375hex.tech | Hex | developer,saas376observablehq.com | Observable | developer,saas377retool.com | Retool | developer,saas378appsmith.com | Appsmith | developer,open-source,saas379bubble.io | Bubble | developer,saas380tooljet.com | ToolJet | developer,open-source,saas381budibase.com | Budibase | developer,open-source,saas | GB382softr.io | Softr | developer,saas | DE383glideapps.com | Glide | developer,saas384adalo.com | Adalo | developer,saas385flutterflow.io | FlutterFlow | developer,saas386workato.com | Workato | enterprise,saas387tray.io | Tray.ai | enterprise,saas388celigo.com | Celigo | enterprise,saas389boomi.com | Boomi | enterprise,saas390informatica.com | Informatica | enterprise,saas391matillion.com | Matillion | developer,saas | GB392alteryx.com | Alteryx | enterprise,saas393dataiku.com | Dataiku | enterprise,ai,saas | FR394domo.com | Domo | enterprise,saas395sisense.com | Sisense | enterprise,saas396thoughtspot.com | ThoughtSpot | enterprise,saas397qlik.com | Qlik | enterprise,saas398microstrategy.com | Strategy (MicroStrategy) | enterprise,saas399gooddata.com | GoodData | enterprise,saas400cube.dev | Cube | developer,open-source,saas401preset.io | Preset | developer,saas402lightdash.com | Lightdash | developer,open-source,saas | GB403evidence.dev | Evidence | developer,open-source404count.co | Count | developer,saas | GB405deepnote.com | Deepnote | developer,saas406anaconda.com | Anaconda | developer,software407comet.com | Comet | developer,ai,saas408neptune.ai | Neptune.ai | developer,ai,saas | PL409tecton.ai | Tecton | developer,ai,saas410prefect.io | Prefect | developer,open-source,saas411dagster.io | Dagster | developer,open-source,saas412astronomer.io | Astronomer | developer,saas413mage.ai | Mage | developer,open-source,saas414kestra.io | Kestra | developer,open-source,saas | FR415temporal.io | Temporal | developer,open-source,saas416trigger.dev | Trigger.dev | developer,open-source,saas417windmill.dev | Windmill | developer,open-source,saas418pipedream.com | Pipedream | developer,saas419val.town | Val Town | developer,saas420montecarlodata.com | Monte Carlo | developer,saas421bigeye.com | Bigeye | developer,saas422soda.io | Soda | developer,saas | BE423atlan.com | Atlan | enterprise,saas | SG424alation.com | Alation | enterprise,saas425collibra.com | Collibra | enterprise,saas | BE426secoda.co | Secoda | enterprise,saas | CA427firebolt.io | Firebolt | developer,saas | IL428motherduck.com | MotherDuck | developer,saas429starburst.io | Starburst | developer,saas430dremio.com | Dremio | developer,saas431imply.io | Imply | developer,saas432timescale.com | Timescale | developer,open-source,saas433influxdata.com | InfluxData | developer,open-source,saas434questdb.io | QuestDB | developer,open-source,saas | GB435singlestore.com | SingleStore | developer,saas436planetscale.com | PlanetScale | developer,saas437yugabyte.com | YugabyteDB | developer,open-source,saas438fauna.com | Fauna | developer,saas439xata.io | Xata | developer,saas | DE440turso.tech | Turso | developer,saas441nhost.io | Nhost | developer,open-source,saas | SE442appwrite.io | Appwrite | developer,open-source,saas443convex.dev | Convex | developer,saas444hasura.io | Hasura | developer,open-source,saas445prisma.io | Prisma | developer,open-source,saas | DE446apollographql.com | Apollo GraphQL | developer,open-source,saas447gomomento.com | Momento | developer,saas448weaviate.io | Weaviate | developer,open-source,ai,saas | NL449qdrant.tech | Qdrant | developer,open-source,ai,saas | DE450zilliz.com | Zilliz | developer,ai,saas451lancedb.com | LanceDB | developer,open-source,ai452vespa.ai | Vespa.ai | developer,open-source,ai | NO453turbopuffer.com | turbopuffer | developer,ai,saas | CA454opensearch.org | OpenSearch | developer,open-source455constructor.io | Constructor | commerce,saas456bloomreach.com | Bloomreach | commerce,saas457# payments, fintech & finance apps458adyen.com | Adyen | payments,finance | NL459checkout.com | Checkout.com | payments,finance | GB460braintreepayments.com | Braintree | payments,developer461squareup.com | Square | payments,commerce462recurly.com | Recurly | payments,saas463chargebee.com | Chargebee | payments,saas464paddle.com | Paddle | payments,saas | GB465lemonsqueezy.com | Lemon Squeezy | payments,saas466gumroad.com | Gumroad | commerce,payments467fastspring.com | FastSpring | payments,saas468gocardless.com | GoCardless | payments,finance | GB469unit.co | Unit | payments,developer470moderntreasury.com | Modern Treasury | payments,developer471ramp.com | Ramp | finance,saas472revolut.com | Revolut | finance,payments | GB473n26.com | N26 | finance | DE474monzo.com | Monzo | finance | GB475starlingbank.com | Starling Bank | finance | GB476sofi.com | SoFi | finance477afterpay.com | Afterpay | payments | AU478sezzle.com | Sezzle | payments479zip.co | Zip | payments | AU480bolt.com | Bolt | payments,commerce481venmo.com | Venmo | payments,consumer-tech482dwolla.com | Dwolla | payments,developer483tipalti.com | Tipalti | payments,saas484bill.com | BILL | payments,saas485melio.com | Melio | payments,saas486navan.com | Navan | enterprise,travel,saas487spendesk.com | Spendesk | finance,saas | FR488pleo.io | Pleo | finance,saas | DK489payhawk.com | Payhawk | finance,saas | GB490coupa.com | Coupa | enterprise,saas491intuit.com | Intuit | finance,saas492quickbooks.intuit.com | QuickBooks | finance,saas493zuora.com | Zuora | payments,saas494maxio.com | Maxio | payments,saas495withorb.com | Orb | payments,developer496metronome.com | Metronome | payments,developer497getlago.com | Lago | payments,open-source,developer | FR498stigg.io | Stigg | payments,developer | IL499revenuecat.com | RevenueCat | payments,developer500adapty.io | Adapty | payments,developer501avalara.com | Avalara | finance,saas502taxjar.com | TaxJar | finance,saas503anrok.com | Anrok | finance,saas504alloy.com | Alloy | finance,cyber,saas505withpersona.com | Persona | cyber,saas506onfido.com | Onfido | cyber,saas | GB507jumio.com | Jumio | cyber,saas508veriff.com | Veriff | cyber,saas | EE509socure.com | Socure | cyber,saas510sardine.ai | Sardine | cyber,payments,saas511sift.com | Sift | cyber,saas512forter.com | Forter | cyber,commerce,saas | IL513riskified.com | Riskified | cyber,commerce,saas | IL514signifyd.com | Signifyd | cyber,commerce,saas515unit21.ai | Unit21 | cyber,finance,saas516chainalysis.com | Chainalysis | crypto,cyber,saas517elliptic.co | Elliptic | crypto,cyber,saas | GB518etoro.com | eToro | finance | IL519interactivebrokers.com | Interactive Brokers | finance520schwab.com | Charles Schwab | finance521fidelity.com | Fidelity Investments | finance522vanguard.com | Vanguard | finance523questrade.com | Questrade | finance | CA524betterment.com | Betterment | finance525wealthfront.com | Wealthfront | finance526acorns.com | Acorns | finance527public.com | Public | finance528alpaca.markets | Alpaca | finance,developer529tradier.com | Tradier | finance,developer530polygon.io | Polygon.io | finance,developer531finnhub.io | Finnhub | finance,developer532twelvedata.com | Twelve Data | finance,developer533intrinio.com | Intrinio | finance,developer534financialmodelingprep.com | Financial Modeling Prep | finance,developer535ynab.com | YNAB | finance,consumer-tech536nerdwallet.com | NerdWallet | finance,media537koho.ca | KOHO | finance | CA538neofinancial.com | Neo Financial | finance | CA539borrowell.com | Borrowell | finance | CA540razorpay.com | Razorpay | payments | IN541paytm.com | Paytm | payments | IN542phonepe.com | PhonePe | payments | IN543nuvei.com | Nuvei | payments | CA544paymentus.com | Paymentus | payments | CA545plooto.com | Plooto | payments | CA546clio.com | Clio | enterprise,legal,saas | CA547jobber.com | Jobber | enterprise,saas | CA548benevity.com | Benevity | enterprise,saas | CA549ada.cx | Ada | enterprise,ai,saas | CA550kinaxis.com | Kinaxis | enterprise,saas | CA551league.com | League | health,saas | CA552tophat.com | Top Hat | education,saas | CA553faire.com | Faire | commerce554# consumer internet555spotify.com | Spotify | entertainment,media | SE556netflix.com | Netflix | entertainment,media557disneyplus.com | Disney+ | entertainment,media558youtube.com | YouTube | media,internet559twitch.tv | Twitch | media,gaming560discord.com | Discord | consumer-tech,gaming561pinterest.com | Pinterest | consumer-tech,internet562snap.com | Snap | consumer-tech,internet563tiktok.com | TikTok | consumer-tech,internet | SG564x.com | X | consumer-tech,internet565threads.net | Threads | consumer-tech,internet566bsky.app | Bluesky | consumer-tech,internet,open-source567joinmastodon.org | Mastodon | consumer-tech,open-source | DE568signal.org | Signal | consumer-tech,cyber,open-source569telegram.org | Telegram | consumer-tech,internet | AE570whatsapp.com | WhatsApp | consumer-tech,internet571peloton.com | Peloton | consumer-tech,health572uber.com | Uber | transport,consumer-tech573lyft.com | Lyft | transport,consumer-tech574doordash.com | DoorDash | food,consumer-tech575instacart.com | Instacart | retail,consumer-tech576airbnb.com | Airbnb | travel,consumer-tech577booking.com | Booking.com | travel | NL578expedia.com | Expedia | travel579hulu.com | Hulu | entertainment,media580max.com | Max | entertainment,media581paramountplus.com | Paramount+ | entertainment,media582peacocktv.com | Peacock | entertainment,media583crunchyroll.com | Crunchyroll | entertainment,media584tidal.com | TIDAL | entertainment,media | NO585deezer.com | Deezer | entertainment,media | FR586soundcloud.com | SoundCloud | entertainment,media | DE587pandora.com | Pandora | entertainment,media588audible.com | Audible | entertainment,media589letterboxd.com | Letterboxd | entertainment,consumer-tech | NZ590fandom.com | Fandom | media,gaming591quora.com | Quora | consumer-tech,internet592tumblr.com | Tumblr | consumer-tech,media593flickr.com | Flickr | consumer-tech,media594dribbble.com | Dribbble | consumer-tech,media595behance.net | Behance | consumer-tech,media596unsplash.com | Unsplash | consumer-tech,media597pexels.com | Pexels | consumer-tech,media | DE598giphy.com | GIPHY | consumer-tech,media599imgur.com | Imgur | consumer-tech,media600wattpad.com | Wattpad | consumer-tech,media | CA601ko-fi.com | Ko-fi | consumer-tech,payments | GB602buymeacoffee.com | Buy Me a Coffee | consumer-tech,payments603kick.com | Kick | media,gaming | AU604rumble.com | Rumble | media | CA605dailymotion.com | Dailymotion | media | FR606nebula.tv | Nebula | media607masterclass.com | MasterClass | education,media608skillshare.com | Skillshare | education,media609edx.org | edX | education610khanacademy.org | Khan Academy | education611brilliant.org | Brilliant | education612codecademy.com | Codecademy | education,developer613datacamp.com | DataCamp | education,developer614pluralsight.com | Pluralsight | education,developer615chegg.com | Chegg | education616quizlet.com | Quizlet | education617blackboard.com | Blackboard (Anthology) | education,saas618moodle.org | Moodle | education,open-source | AU619d2l.com | D2L | education,saas | CA620clever.com | Clever | education,saas621classdojo.com | ClassDojo | education,consumer-tech622remind.com | Remind | education,saas623headspace.com | Headspace | health,consumer-tech624calm.com | Calm | health,consumer-tech625noom.com | Noom | health,consumer-tech626myfitnesspal.com | MyFitnessPal | health,consumer-tech627fitbit.com | Fitbit | health,consumer-tech628garmin.com | Garmin | consumer-tech,transport | CH629withings.com | Withings | health,consumer-tech | FR630zwift.com | Zwift | health,gaming631alltrails.com | AllTrails | consumer-tech,travel632komoot.com | komoot | consumer-tech,travel | DE633transitapp.com | Transit | transport,consumer-tech | CA634moovit.com | Moovit | transport,consumer-tech | IL635lime.co | Lime | transport636bolt.eu | Bolt | transport | EE637grab.com | Grab | transport,payments | SG638careem.com | Careem | transport | AE639blablacar.com | BlaBlaCar | transport | FR640flixbus.com | FlixBus | transport | DE641turo.com | Turo | transport,travel642getaround.com | Getaround | transport643zipcar.com | Zipcar | transport644grubhub.com | Grubhub | food,consumer-tech645deliveroo.com | Deliveroo | food,consumer-tech | GB646justeattakeaway.com | Just Eat Takeaway.com | food,consumer-tech | NL647skipthedishes.com | SkipTheDishes | food,consumer-tech | CA648gopuff.com | Gopuff | retail,consumer-tech649getir.com | Getir | retail,consumer-tech | TR650glovoapp.com | Glovo | food,consumer-tech | ES651wolt.com | Wolt | food,consumer-tech | FI652hellofresh.com | HelloFresh | food,consumer-tech | DE653vrbo.com | Vrbo | travel654hopper.com | Hopper | travel | CA655kayak.com | KAYAK | travel656skyscanner.net | Skyscanner | travel | GB657tripadvisor.com | Tripadvisor | travel658trivago.com | trivago | travel | DE659hostelworld.com | Hostelworld | travel | IE660agoda.com | Agoda | travel | SG661trip.com | Trip.com | travel | SG662marriott.com | Marriott | travel663hilton.com | Hilton | travel664experian.com | Experian | finance | IE665equifax.com | Equifax | finance666transunion.com | TransUnion | finance667creditkarma.com | Credit Karma | finance,consumer-tech668# games669store.steampowered.com | Steam | gaming670xbox.com | Xbox | gaming671nintendo.com | Nintendo | gaming | JP672riotgames.com | Riot Games | gaming673blizzard.com | Blizzard Entertainment | gaming674ea.com | Electronic Arts | gaming675ubisoft.com | Ubisoft | gaming | FR676roblox.com | Roblox | gaming677minecraft.net | Minecraft | gaming678unity.com | Unity | gaming,developer679unrealengine.com | Unreal Engine | gaming,developer680godotengine.org | Godot Engine | gaming,open-source681itch.io | itch.io | gaming682gog.com | GOG | gaming | PL683humblebundle.com | Humble Bundle | gaming684supercell.com | Supercell | gaming | FI685rockstargames.com | Rockstar Games | gaming686bungie.net | Bungie | gaming687squareenix.com | Square Enix | gaming | JP688bethesda.net | Bethesda | gaming689cdprojektred.com | CD Projekt Red | gaming | PL690krafton.com | Krafton | gaming | KR691nexon.com | Nexon | gaming | JP692miHoYo.com | miHoYo | gaming | CN693hoyoverse.com | HoYoverse | gaming | SG694scopely.com | Scopely | gaming695zynga.com | Zynga | gaming696kingdom.com | King | gaming | SE697playtika.com | Playtika | gaming | IL698paradoxplaza.com | Paradox Interactive | gaming | SE699frontier.co.uk | Frontier Developments | gaming | GB700larian.com | Larian Studios | gaming | BE701mojang.com | Mojang | gaming | SE702# security / IT tooling saas (long tail)703cyberark.com | CyberArk | cyber,saas | IL704beyondtrust.com | BeyondTrust | cyber,saas705delinea.com | Delinea | cyber,saas706duo.com | Cisco Duo | cyber,saas707yubico.com | Yubico | cyber | SE708proofpoint.com | Proofpoint | cyber,saas709mimecast.com | Mimecast | cyber,saas | GB710barracuda.com | Barracuda Networks | cyber,saas711zscaler.com | Zscaler | cyber,saas712netskope.com | Netskope | cyber,saas713cloudflare.com | Cloudflare | cloud,cyber714rapid7.com | Rapid7 | cyber,saas715qualys.com | Qualys | cyber,saas716arcticwolf.com | Arctic Wolf | cyber,saas717huntress.com | Huntress | cyber,saas718kandji.io | Kandji | cyber,enterprise,saas719jamf.com | Jamf | enterprise,saas720addigy.com | Addigy | enterprise,saas721mosyle.com | Mosyle | enterprise,saas722ninjaone.com | NinjaOne | enterprise,saas723atera.com | Atera | enterprise,saas | IL724connectwise.com | ConnectWise | enterprise,saas725kaseya.com | Kaseya | enterprise,saas726datto.com | Datto | enterprise,saas727veeam.com | Veeam | enterprise,software | CH728druva.com | Druva | enterprise,saas729backblaze.com | Backblaze | cloud,consumer-tech730carbonite.com | Carbonite | consumer-tech,saas731idrive.com | IDrive | consumer-tech,saas732pcloud.com | pCloud | consumer-tech,saas | CH733sync.com | Sync.com | consumer-tech,saas | CA734mega.io | MEGA | consumer-tech,saas | NZ735icedrive.net | Icedrive | consumer-tech,saas | GB736nordvpn.com | NordVPN | cyber,consumer-tech | PA737expressvpn.com | ExpressVPN | cyber,consumer-tech | VG738mullvad.net | Mullvad | cyber,consumer-tech | SE739protonvpn.com | Proton VPN | cyber,consumer-tech | CH740surfshark.com | Surfshark | cyber,consumer-tech | NL741tunnelbear.com | TunnelBear | cyber,consumer-tech | CA742"""743744FEED_PATHS = [745 "/changelog/rss.xml", "/changelog/feed.xml", "/changelog.rss", "/changelog/feed", "/changelog/rss",746 "/releases/feed", "/release-notes/rss.xml", "/updates/rss.xml",747 "/blog/rss.xml", "/blog/feed", "/blog/feed.xml", "/blog/rss",748 "/feed.xml", "/rss.xml", "/feed", "/atom.xml",749]750SUBDOMAIN_FEEDS = [("blog", ["/feed", "/rss.xml", "/feed.xml"]), ("changelog", ["/feed.xml", "/rss.xml"])]751HTML_PAGES = [("/changelog", "changelog", "B"), ("/pricing", "pricing", "C")]752STATUS_JSON_PATHS = ["/api/v2/summary.json", "/summary.json", "/api/v1/summary"]753STATUS_FEED_PATHS = ["/history.rss", "/history.atom", "/feed", "/rss", "/feed.rss", "/index.rss", "/history.xml", "/api/v2/incidents.rss"]754CHALLENGE = re.compile(r"just a moment|cf-chl|attention required|access denied|captcha|enable javascript and cookies|verify you are human|ddos-guard|incapsula|_incapsula", re.I)755756757class Bucket:758 def __init__(self, rate: float) -> None:759 self.rate, self.tokens, self.t, self.lock = rate, 1.0, time.monotonic(), threading.Lock()760761 def take(self) -> None:762 while True:763 with self.lock:764 now = time.monotonic()765 self.tokens = min(1.0 + self.rate, self.tokens + (now - self.t) * self.rate)766 self.t = now767 if self.tokens >= 1:768 self.tokens -= 1769 return770 wait = (1 - self.tokens) / self.rate771 time.sleep(wait)772773774BUCKET = Bucket(RATE)775_local = threading.local()776_dns_cache: dict[str, bool] = {}777_dns_lock = threading.Lock()778779780def session() -> requests.Session:781 s = getattr(_local, "s", None)782 if s is None:783 s = requests.Session()784 s.headers.update({"User-Agent": UA, "Accept": "application/json, application/rss+xml, application/atom+xml, application/xml, text/xml, text/html;q=0.9, */*;q=0.8", "Accept-Language": "en"})785 _local.s = s786 return s787788789def resolves(host: str) -> bool:790 with _dns_lock:791 if host in _dns_cache:792 return _dns_cache[host]793 try:794 socket.getaddrinfo(host, 443, proto=socket.IPPROTO_TCP)795 ok = True796 except OSError:797 ok = False798 with _dns_lock:799 _dns_cache[host] = ok800 return ok801802803def get(url: str) -> dict | None:804 host = urlparse(url).hostname or ""805 if not resolves(host):806 return None807 BUCKET.take()808 try:809 r = session().get(url, timeout=TIMEOUT, allow_redirects=True, stream=True)810 body = b""811 for chunk in r.iter_content(65536):812 body += chunk813 if len(body) > MAX_BODY:814 break815 r.close()816 text = body.decode(r.encoding or "utf-8", errors="replace") if body else ""817 return {"status": r.status_code, "ct": (r.headers.get("content-type") or "").lower(), "final": r.url, "text": text, "len": len(body)}818 except requests.RequestException:819 return None820821822def json_shape(text: str) -> tuple[str, dict] | None:823 t = text.lstrip(" \n\r\t")824 if not t.startswith("{"):825 return None826 try:827 j = json.loads(t)828 except ValueError:829 return None830 if not isinstance(j, dict):831 return None832 page = j.get("page")833 if isinstance(page, dict) and isinstance(j.get("status"), dict) and ("components" in j or "incidents" in j) and "id" in page:834 return "statuspage", j835 if isinstance(j.get("subpages"), list) and any(isinstance(p, dict) and "summary" in p for p in j["subpages"]):836 return "incidentio", j837 if isinstance(page, dict) and ("activeIncidents" in j or "activeMaintenances" in j or isinstance(page.get("status"), str)):838 return "instatus", j839 if "ongoing_incidents" in j or "page_title" in j:840 return "incidentio", j841 if isinstance(j.get("result"), dict) and "status_overall" in j["result"]:842 return "statusio", j843 return None844845846def feed_shape(text: str, ct: str) -> dict | None:847 head = text[:4000].lstrip(" \n\r\t")848 if "text/html" in ct and not head.startswith("<?xml") and not re.match(r"<(rss|feed|rdf:RDF)\b", head):849 return None850 if not re.search(r"<(rss|feed|rdf:RDF)[\s>]", head):851 return None852 items = len(re.findall(r"<(item|entry)[\s>]", text))853 if items == 0:854 return None855 title = re.search(r"<title[^>]*>(?:<!\[CDATA\[)?(.*?)(?:\]\]>)?</title>", text, re.S)856 dates = re.findall(r"<(?:pubDate|updated|published|dc:date)[^>]*>([^<]+)<", text)857 return {"items": items, "title": (title.group(1).strip()[:80] if title else ""), "latest": dates[0].strip() if dates else "", "hash": hashlib.sha1(re.sub(r"\s+", " ", text[:20000]).encode()).hexdigest()[:12]}858859860def html_text_len(text: str) -> int:861 t = re.sub(r"<(script|style|noscript|svg)[^>]*>.*?</\1>", " ", text, flags=re.S | re.I)862 t = re.sub(r"<[^>]+>", " ", t)863 t = re.sub(r"\s+", " ", t)864 return len(t)865866867def parse_candidates() -> list[dict]:868 out, seen = [], set()869 for ln in CANDIDATES.strip().splitlines():870 ln = ln.strip()871 if not ln or ln.startswith("#"):872 continue873 parts = [p.strip() for p in ln.split("|")]874 domain = parts[0].lower()875 if domain in seen:876 continue877 seen.add(domain)878 out.append({"domain": domain, "name": parts[1], "categories": [c.strip() for c in parts[2].split(",")], "country": parts[3] if len(parts) > 3 else "US"})879 return out880881882def load_registry() -> tuple[dict[str, str], set[str], set[str], set[str]]:883 """id→domain, all sensor URLs, hosts of existing status sensors, all ids."""884 id_domain: dict[str, str] = {}885 urls: set[str] = set()886 status_hosts: set[str] = set()887 ids: set[str] = set()888 for f in ["config/sources.yaml"] + sorted(glob.glob("config/sources.d/*.yaml")):889 if f.endswith("48-saas-status-changelogs.yaml"):890 continue891 d = yaml.safe_load(open(f)) or {}892 for s in d.get("sources", []):893 ids.add(s["id"])894 if s.get("domain"):895 id_domain[s["id"]] = s["domain"].lower()896 for sen in s.get("sensors", []) or []:897 u = sen.get("url", "")898 urls.add(u.rstrip("/"))899 if sen.get("type") == "STATUSPAGE" or sen.get("connector") in ("statuspage", "statusjson") or "status" in (sen.get("name") or ""):900 h = urlparse(u).hostname901 if h:902 status_hosts.add(h)903 return id_domain, urls, status_hosts, ids904905906def apex(host: str) -> str:907 parts = host.split(".")908 if len(parts) >= 3 and parts[-2] in ("co", "com", "org", "gov", "ac", "net") and len(parts[-1]) == 2:909 return ".".join(parts[-3:])910 return ".".join(parts[-2:])911912913def probe_domain(c: dict, existing_urls: set[str], status_hosts: set[str]) -> dict:914 d = c["domain"]915 hits: list[dict] = []916 notes: list[str] = []917 probed_hosts: set[str] = set()918919 def try_status_host(host: str, label: str) -> bool:920 if host in probed_hosts:921 return False922 if host in status_hosts:923 notes.append(f"{host}: already covered")924 return True925 probed_hosts.add(host)926 for p in STATUS_JSON_PATHS:927 url = f"https://{host}{p}"928 if url in existing_urls:929 return True930 if not resolves(host):931 return False932 r = get(url)933 if not r:934 continue935 if r["status"] == 200:936 sh = json_shape(r["text"])937 if sh:938 flavor, j = sh939 title = (j.get("page") or {}).get("name") if isinstance(j.get("page"), dict) else j.get("page_title", "")940 hits.append({"kind": "status", "flavor": flavor, "url": url, "final": r["final"], "title": title or "", "via": label})941 return True942 # HTML root → provider sniffing943 r = get(f"https://{host}/")944 if r and r["status"] == 200:945 body = r["text"]946 m = re.search(r"statusio_page_id['\"]?\s*[:=]\s*['\"]([0-9a-f]{20,})", body) or re.search(r"api\.status\.io/1\.0/status/([0-9a-f]{20,})", body)947 if m:948 url = f"https://api.status.io/1.0/status/{m.group(1)}"949 rr = get(url)950 if rr and rr["status"] == 200 and json_shape(rr["text"]):951 hits.append({"kind": "status", "flavor": "statusio", "url": url, "final": rr["final"], "title": "", "via": label})952 return True953 prov = "hund" if "hund.io" in body else "sorry" if "sorryapp" in body else "firehydrant" if "firehydrant" in body else "betterstack" if ("betteruptime" in body or "betterstack" in body) else "freshstatus" if "freshstatus" in body else "statuspage-html" if "statuspage.io" in body else "custom"954 if prov == "betterstack":955 notes.append(f"{host}: Better Stack status page (no JSON API)")956 return False957 for p in STATUS_FEED_PATHS:958 url = f"https://{host}{p}"959 if url in existing_urls:960 return True961 rr = get(url)962 if rr and rr["status"] == 200:963 fs = feed_shape(rr["text"], rr["ct"])964 if fs:965 hits.append({"kind": "status-feed", "flavor": prov, "url": url, "final": rr["final"], "title": fs["title"], "items": fs["items"], "latest": fs["latest"], "via": label})966 return True967 notes.append(f"{host}: {prov} status page, no machine-readable endpoint")968 elif r and r["status"] in (403, 429, 503):969 notes.append(f"{host}: http {r['status']}")970 return False971972 # 1. status.<domain> (and where its root redirects)973 sh = f"status.{apex(d)}"974 if sh not in status_hosts:975 found = False976 if resolves(sh):977 root = get(f"https://{sh}/")978 if root:979 fh = urlparse(root["final"]).hostname or sh980 found = try_status_host(sh, "status host")981 if not found and fh != sh:982 found = try_status_host(fh, f"redirect from {sh}")983 else:984 notes.append(f"{sh}: unreachable")985 # 2. <domain>/status redirecting to a status host986 if not found:987 r = get(f"https://{d}/status")988 if r and r["status"] == 200:989 fh = urlparse(r["final"]).hostname or d990 fp = urlparse(r["final"]).path991 if fh not in (d, f"www.{d}") and fh not in probed_hosts:992 try_status_host(fh, f"redirect from {d}/status")993 elif fp.rstrip("/").endswith("/status") and json_shape(r["text"]) is None and "text/html" in r["ct"]:994 if "statuspage.io" in r["text"] or "instatus" in r["text"]:995 m = re.search(r"https://([a-z0-9.-]+)/api/v2/", r["text"])996 if m:997 try_status_host(m.group(1), f"embedded in {d}/status")998 else:999 notes.append(f"{sh}: already covered")10001001 # 3. feeds on the main domain1002 feed_hits = 01003 seen_final: set[str] = set()1004 seen_hash: set[str] = set()1005 dead = 01006 for p in FEED_PATHS:1007 if feed_hits >= 2 or dead >= 3:1008 break1009 url = f"https://{d}{p}"1010 if url in existing_urls:1011 continue1012 r = get(url)1013 if not r:1014 dead += 11015 continue1016 if r["status"] in (403, 429) and p == FEED_PATHS[0]:1017 notes.append(f"{d}: http {r['status']} (blocked)")1018 dead = 31019 break1020 if r["status"] != 200:1021 continue1022 fs = feed_shape(r["text"], r["ct"])1023 if not fs:1024 continue1025 fk = r["final"].rstrip("/")1026 if fk in seen_final or fs["hash"] in seen_hash or fk in existing_urls:1027 continue1028 seen_final.add(fk)1029 seen_hash.add(fs["hash"])1030 hits.append({"kind": "feed", "path": p, "url": url, "final": r["final"], "title": fs["title"], "items": fs["items"], "latest": fs["latest"]})1031 feed_hits += 11032 # 4. blog./changelog. subdomains1033 for sub, paths in SUBDOMAIN_FEEDS:1034 host = f"{sub}.{apex(d)}"1035 if not resolves(host):1036 continue1037 for p in paths:1038 url = f"https://{host}{p}"1039 if url in existing_urls:1040 break1041 r = get(url)1042 if not r or r["status"] != 200:1043 continue1044 fs = feed_shape(r["text"], r["ct"])1045 if not fs:1046 continue1047 fk = r["final"].rstrip("/")1048 if fk in seen_final or fs["hash"] in seen_hash or fk in existing_urls:1049 break1050 seen_final.add(fk)1051 seen_hash.add(fs["hash"])1052 hits.append({"kind": "feed", "path": f"{sub}:{p}", "url": url, "final": r["final"], "title": fs["title"], "items": fs["items"], "latest": fs["latest"]})1053 break1054 # 5. server-rendered HTML pages1055 for p, label, tier in HTML_PAGES:1056 url = f"https://{d}{p}"1057 if url in existing_urls or f"https://www.{d}{p}" in existing_urls:1058 continue1059 r = get(url)1060 if not r or r["status"] != 200 or "text/html" not in r["ct"]:1061 continue1062 fpath = urlparse(r["final"]).path.rstrip("/")1063 if fpath in ("", "/") or CHALLENGE.search(r["text"][:6000]):1064 continue1065 n = html_text_len(r["text"])1066 if n >= 2500:1067 hits.append({"kind": "html", "label": label, "tier": tier, "url": url, "final": r["final"], "textlen": n})1068 return {"domain": d, "hits": hits, "notes": notes}106910701071def cmd_probe(args: argparse.Namespace) -> None:1072 id_domain, urls, status_hosts, _ids = load_registry()1073 cands = parse_candidates()1074 if args.only:1075 only = set(args.only.split(","))1076 cands = [c for c in cands if c["domain"] in only]1077 done: set[str] = set()1078 if os.path.exists(args.out) and not args.fresh:1079 for ln in open(args.out):1080 try:1081 done.add(json.loads(ln)["domain"])1082 except ValueError:1083 pass1084 todo = [c for c in cands if c["domain"] not in done]1085 print(f"{len(cands)} candidates, {len(done)} already probed, {len(todo)} to go · ≤ {RATE} req/s", file=sys.stderr)1086 lock = threading.Lock()1087 t0 = time.time()1088 with open(args.out, "a") as out, ThreadPoolExecutor(max_workers=args.workers) as ex:1089 futs = {ex.submit(probe_domain, c, urls, status_hosts): c for c in todo}1090 for i, fut in enumerate(as_completed(futs), 1):1091 c = futs[fut]1092 try:1093 res = fut.result()1094 except Exception as e: # noqa: BLE0011095 res = {"domain": c["domain"], "hits": [], "notes": [f"probe error: {e}"]}1096 res.update({"name": c["name"], "categories": c["categories"], "country": c["country"]})1097 with lock:1098 out.write(json.dumps(res) + "\n")1099 out.flush()1100 kinds = ",".join(sorted({h["kind"] + (":" + h["flavor"] if h.get("flavor") else "") for h in res["hits"]})) or "-"1101 print(f"[{i}/{len(todo)} {time.time() - t0:5.0f}s] {c['domain']:28s} {kinds}", file=sys.stderr)110211031104def slug(s: str) -> str:1105 s = s.lower().replace("&", " and ")1106 s = re.sub(r"\(.*?\)", "", s)1107 s = re.sub(r"[^a-z0-9]+", "-", s).strip("-")1108 return s110911101111# Same organization declared earlier under another domain → extend instead of redeclaring.1112EXTEND_OVERRIDE = {"deliveroo.com": "deliveroo", "greenhouse.io": "greenhouse", "notion.so": "notion", "peloton.com": "peloton",1113 "timescale.com": "timescale", "sparkpost.com": "bird", "unrealengine.com": "epic-games"}1114# Different organization whose natural slug is taken (bolt = Bolt EE ride-hailing, census = US Census Bureau).1115NEW_ID_OVERRIDE = {"bolt.com": "bolt-payments", "getcensus.com": "getcensus"}111611171118def cmd_emit(args: argparse.Namespace) -> None:1119 id_domain, urls, status_hosts, ids = load_registry()1120 domain_id = dict(EXTEND_OVERRIDE)1121 for i, dom in id_domain.items():1122 domain_id.setdefault(dom, i)1123 domain_id.setdefault(apex(dom), i)1124 rows = [json.loads(ln) for ln in open(args.inp) if ln.strip()]1125 rows.sort(key=lambda r: r["domain"])1126 used_urls: set[str] = set()1127 new_ids: set[str] = set()1128 entries: dict[str, tuple[list[str], list[str]]] = {}1129 print("# config/sources.d/48-saas-status-changelogs.yaml — 2026-09-11 — the long tail of SaaS / B2B / consumer-internet")1130 print("# services: machine-readable status pages (Atlassian Statuspage summary API, Instatus summary.json, incident.io")1131 print("# /api/v1/summary, Status.io, RSS history feeds of Hund/Sorry/FireHydrant pages), changelog / release-note / blog")1132 print("# feeds and server-rendered changelog / pricing pages. Generated by scripts/gen-status-probe.py (probe of ~600")1133 print("# candidate domains at ≤ 4 req/s, 2026-09-11), curated by hand, then pruned with the live validator: every sensor")1134 print("# below was fetched and parsed by apps/engine/src/validate.ts. Organizations already declared in files 10–46 are")1135 print("# extended (`extend: true`); the others are declared here with domain, categories, tier and country.")1136 print("sources:")1137 for r in rows:1138 if not r["hits"]:1139 continue1140 d = r["domain"]1141 existing = domain_id.get(d) or domain_id.get(apex(d)) or domain_id.get(f"www.{d}")1142 sensors: list[str] = []1143 has_status = False1144 for h in r["hits"]:1145 u = h["url"].rstrip("/")1146 if u in urls or u in used_urls or h["final"].rstrip("/") in urls:1147 continue1148 used_urls.add(u)1149 if h["kind"] == "status":1150 has_status = True1151 if h["flavor"] == "statuspage":1152 sensors.append(f' - {{ name: status, url: "{h["url"]}", type: STATUSPAGE, connector: statuspage, tier: S }}')1153 else:1154 label = {"instatus": "instatus", "incidentio": "incident.io", "statusio": "status.io"}[h["flavor"]]1155 sensors.append(f' - {{ name: status ({label}), url: "{h["url"]}", type: STATUSPAGE, connector: statusjson, tier: S }}')1156 elif h["kind"] == "status-feed":1157 sensors.append(f' - {{ name: status history feed, url: "{h["url"]}", type: RSS, connector: rss, tier: A }}')1158 elif h["kind"] == "feed":1159 p = h.get("path", "")1160 name = "changelog feed" if "changelog" in p or "release" in p or "updates" in p or "whats-new" in p or "product-updates" in p else "blog feed" if "blog" in p else "news feed" if "news" in p else "feed"1161 cap = ", config: { maxItems: 60 }" if (h.get("items") or 0) > 60 else ""1162 sensors.append(f' - {{ name: {name}, url: "{h["url"]}", type: {"ATOM" if p.endswith("atom.xml") else "RSS"}, connector: rss, tier: B{cap} }}')1163 elif h["kind"] == "html":1164 sensors.append(f' - {{ name: {h["label"]}, url: "{h["url"]}", type: HTML, connector: http, tier: {h["tier"]} }}')1165 if not sensors:1166 continue1167 # dedupe sensor names within the source (feed vs feed)1168 seen_names: dict[str, int] = {}1169 fixed = []1170 for s in sensors:1171 m = re.match(r" - \{ name: ([^,]+),", s)1172 nm = m.group(1)1173 if nm in seen_names:1174 seen_names[nm] += 11175 s = s.replace(f"name: {nm},", f"name: {nm} {seen_names[nm]},", 1)1176 else:1177 seen_names[nm] = 11178 fixed.append(s)1179 if existing:1180 head = [f" - id: {existing}", " extend: true"]1181 if r.get("country"):1182 head.append(f" country: {r['country']}")1183 head.append(" sensors:")1184 key = existing1185 else:1186 sid = NEW_ID_OVERRIDE.get(d) or slug(r["name"]) or slug(d.split(".")[0])1187 if sid in ids or sid in new_ids:1188 sid = slug(d.replace(".", "-"))1189 if sid in ids or sid in new_ids:1190 sid = f"{sid}-saas"1191 new_ids.add(sid)1192 cats = ", ".join(r["categories"])1193 head = [f" - id: {sid}", f" name: {r['name']}", f" domain: {d}", f" categories: [{cats}]", f" tier: {'S' if has_status else 'B'}",1194 f" country: {r['country']}", " discover: { rss: true, status: true }", " sensors:"]1195 key = sid1196 if key in entries: # same organization reached through two domains → merge the sensor lists1197 entries[key][1].extend(fixed)1198 else:1199 entries[key] = (head, list(fixed))1200 for head, sens in entries.values():1201 for ln in head:1202 print(ln)1203 for s in sens:1204 print(s)1205 # notes for the header1206 blocked = [n for r in rows for n in r["notes"] if "already covered" not in n and "unreachable" not in n]1207 print("# ── probe notes (not sensors) ──", file=sys.stderr)1208 for n in sorted(set(blocked)):1209 print(f"# {n}", file=sys.stderr)121012111212def main() -> None:1213 ap = argparse.ArgumentParser()1214 sub = ap.add_subparsers(dest="cmd", required=True)1215 p = sub.add_parser("probe")1216 p.add_argument("--out", default="/tmp/probe48.jsonl")1217 p.add_argument("--only")1218 p.add_argument("--fresh", action="store_true")1219 p.add_argument("--workers", type=int, default=8)1220 p.set_defaults(fn=cmd_probe)1221 e = sub.add_parser("emit")1222 e.add_argument("--in", dest="inp", default="/tmp/probe48.jsonl")1223 e.set_defaults(fn=cmd_emit)1224 a = ap.parse_args()1225 a.fn(a)122612271228if __name__ == "__main__":1229 main()1230