Breakage detection only on full re-extraction; first-run crawls each connector once
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
2 changed files +6 −7
modified
deploy/first-run.sh
+2 −6
@@ -12,12 +12,8 @@ AIA=.venv/bin/aia | |||||
| 12 | 12 | ||||
| 13 | $AIA migrate | tail -1 | 13 | $AIA migrate | tail -1 | ||
| 14 | $AIA seed | tail -1 | 14 | $AIA seed | tail -1 | ||
| 15 | −echo "== priority 0 connectors" | 15 | +echo "== all enabled connectors, once each, in priority order" | ||
| 16 | −$AIA crawl --priority 0 || true | 16 | +$AIA crawl --priority 9 || true | ||
| 17 | −echo "== priority 1 connectors" | ||||
| 18 | −$AIA crawl --priority 1 || true | ||||
| 19 | −echo "== priority 2 connectors" | ||||
| 20 | −$AIA crawl --priority 2 || true | ||||
| 21 | $AIA quality | tail -1 | 17 | $AIA quality | tail -1 | ||
| 22 | $AIA stats | head -40 | 18 | $AIA stats | head -40 | ||
| 23 | $AIA status | 19 | $AIA status | ||
modified
src/aiatlas/sdk/connector.py
+4 −1
@@ -225,7 +225,10 @@ class BaseConnector: | |||||
| 225 | 225 | ||||
| 226 | def _final_status(self, ctx: RunContext) -> str: | 226 | def _final_status(self, ctx: RunContext) -> str: | ||
| 227 | s = ctx.stats | 227 | s = ctx.stats | ||
| 228 | − if self.expected_min_records and s.records < self.expected_min_records and not ctx.reprocess: | 228 | + # Breakage detection only makes sense when every fetched document was (re)extracted: unchanged documents legitimately | ||
| 229 | + # produce zero records (304 / same hash), so an incremental run is never "suspect". | ||||
| 230 | + full_extraction = s.docs_fetched > 0 and s.docs_changed >= s.docs_fetched - s.docs_failed and s.docs_unchanged == 0 | ||||
| 231 | + if self.expected_min_records and full_extraction and s.records < self.expected_min_records and not ctx.reprocess: | ||||
| 229 | return "suspect" | 232 | return "suspect" | ||
| 230 | if s.docs_fetched and s.docs_failed == s.docs_fetched and s.docs_fetched > 0: | 233 | if s.docs_fetched and s.docs_failed == s.docs_fetched and s.docs_fetched > 0: | ||
| 231 | return "failed" if s.docs_changed == 0 else "success" | 234 | return "failed" if s.docs_changed == 0 else "success" | ||
| 232 | 235 | ||||