Breakage detection only on full re-extraction; first-run crawls each connector once
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
2 changed files +6 −7
modified
deploy/first-run.sh
+2 −6
@@ -12,12 +12,8 @@ AIA=.venv/bin/aia | ||
| 12 | 12 | |
| 13 | 13 | $AIA migrate | tail -1 |
| 14 | 14 | $AIA seed | tail -1 |
| 15 | −echo "== priority 0 connectors" | |
| 16 | −$AIA crawl --priority 0 || true | |
| 17 | −echo "== priority 1 connectors" | |
| 18 | −$AIA crawl --priority 1 || true | |
| 19 | −echo "== priority 2 connectors" | |
| 20 | −$AIA crawl --priority 2 || true | |
| 15 | +echo "== all enabled connectors, once each, in priority order" | |
| 16 | +$AIA crawl --priority 9 || true | |
| 21 | 17 | $AIA quality | tail -1 |
| 22 | 18 | $AIA stats | head -40 |
| 23 | 19 | $AIA status |
modified
src/aiatlas/sdk/connector.py
+4 −1
@@ -225,7 +225,10 @@ class BaseConnector: | ||
| 225 | 225 | |
| 226 | 226 | def _final_status(self, ctx: RunContext) -> str: |
| 227 | 227 | s = ctx.stats |
| 228 | − if self.expected_min_records and s.records < self.expected_min_records and not ctx.reprocess: | |
| 228 | + # Breakage detection only makes sense when every fetched document was (re)extracted: unchanged documents legitimately | |
| 229 | + # produce zero records (304 / same hash), so an incremental run is never "suspect". | |
| 230 | + full_extraction = s.docs_fetched > 0 and s.docs_changed >= s.docs_fetched - s.docs_failed and s.docs_unchanged == 0 | |
| 231 | + if self.expected_min_records and full_extraction and s.records < self.expected_min_records and not ctx.reprocess: | |
| 229 | 232 | return "suspect" |
| 230 | 233 | if s.docs_fetched and s.docs_failed == s.docs_fetched and s.docs_fetched > 0: |
| 231 | 234 | return "failed" if s.docs_changed == 0 else "success" |
| 232 | 235 | |