SPB Git
spboucher.ai

spb/rareindex

Public

mtggoldfish: XHR headers for price history, api-only engine (no fallback credits)

Simon-Pierre Boucher committed (Sep 7, 2026) parent 9c07cd5 Browse files

2 changed files +282 −2

2 changed files
Unified Split
modified connectors/api/mtggoldfish/index.ts +4 −2
@@ -72,7 +72,7 @@ export function parseSetPage(html: string): { setName: string | null; cards: Car
72 function titleOf(html: string): string | null { 72 function titleOf(html: string): string | null {
73 const m = html.match(/<title>([^<]+)<\/title>/); 73 const m = html.match(/<title>([^<]+)<\/title>/);
74 if (!m) return null; 74 if (!m) return null;
75− return m[1]!.replace(/\s*[|·-]\s*MTGGoldfish.*$/i, '').replace(/\s*Price(s| Guide).*$/i, '').replace(/\s*\([A-Z0-9]{2,6}\)\s*$/, '').trim() || null; 75+ return m[1]!.replace(/\s*[|·-]\s*MTGGoldfish.*$/i, '').replace(/\s*Price(s| Guide).*$/i, '').replace(/\s*Set Information.*$/i, '').replace(/\s*\([A-Z0-9]{2,6}\)\s*$/, '').trim() || null;
76 } 76 }
77 77
78 /** "d += '2010-11-02, 3102.99\n'" lines → [[date, price], …] */ 78 /** "d += '2010-11-02, 3102.99\n'" lines → [[date, price], …] */
@@ -131,7 +131,9 @@ export class MtgGoldfishConnector extends BaseConnector {
131 if (hist.enabled !== false && card.paper !== null && card.paper >= Number(hist.minPrice ?? 100) && historyFetched < Number(hist.maxPerRun ?? 150) && !card.foil) { 131 if (hist.enabled !== false && card.paper !== null && card.paper >= Number(hist.minPrice ?? 100) && historyFetched < Number(hist.maxPerRun ?? 150) && !card.foil) {
132 historyFetched++; 132 historyFetched++;
133 const q = new URLSearchParams({ card_id: card.name, selector: '#tab-paper', type: 'paper', price_type: '' }); 133 const q = new URLSearchParams({ card_id: card.name, selector: '#tab-paper', type: 'paper', price_type: '' });
134− const h = await this.page(ctx, `${BASE}/price_history_component?${q.toString()}`); 134+ // The component is an XHR endpoint: needs the XHR headers and must never fall back to Firecrawl (406 otherwise).
135+ await this.throttle();
136+ const h = await ctx.fetch(`${BASE}/price_history_component?${q.toString()}`, { engines: ['api'], headers: { ...HEADERS, accept: 'text/javascript, application/javascript, */*;q=0.1', 'x-requested-with': 'XMLHttpRequest' }, responseType: 'text', minQuality: 0.3 });
135 if (h.success && h.html) { 137 if (h.success && h.html) {
136 const cutoff = Date.now() - Number(hist.days ?? 1095) * 86_400_000; 138 const cutoff = Date.now() - Number(hist.days ?? 1095) * 86_400_000;
137 const series = parseHistory(h.html).filter(([d]) => new Date(`${d}T00:00:00Z`).getTime() >= cutoff); 139 const series = parseHistory(h.html).filter(([d]) => new Date(`${d}T00:00:00Z`).getTime() >= cutoff);
added data/fixtures/mtggoldfish/lea-history.json +278 −0
@@ -0,0 +1,278 @@
1+{
2+ "raw": {
3+ "url": "https://www.mtggoldfish.com/price/limited-edition-alpha/47/ancestral-recall",
4+ "externalId": "019ce2ef-3ce1-7be7-9635-bb955eb152bd:history",
5+ "kind": "price_observation",
6+ "engine": "api",
7+ "httpStatus": 200,
8+ "payload": {
9+ "kind": "history",
10+ "setName": "Limited Edition Alpha",
11+ "card": {
12+ "name": "Ancestral Recall [LEA]",
13+ "set": "LEA",
14+ "display_name": "Ancestral Recall",
15+ "rarity": "Rare",
16+ "foil": false,
17+ "card_num": 47,
18+ "finish": "regular",
19+ "card_uuid": "019ce2ef-3ce1-7be7-9635-bb955eb152bd",
20+ "paper": 9959.99,
21+ "online": 123.48,
22+ "link": "/price/limited-edition-alpha/47/ancestral-recall",
23+ "image": "https://cards.mtggoldfish.com/images/019ce2ef-3ce1-7be7-9635-bb955eb152bd/variants/672/938/card_image.webp"
24+ },
25+ "series": [
26+ [
27+ "2026-07-08",
28+ 9959.99
29+ ],
30+ [
31+ "2026-07-09",
32+ 9959.99
33+ ],
34+ [
35+ "2026-07-10",
36+ 9959.99
37+ ],
38+ [
39+ "2026-07-11",
40+ 9959.99
41+ ],
42+ [
43+ "2026-07-13",
44+ 9959.99
45+ ],
46+ [
47+ "2026-07-14",
48+ 9959.99
49+ ],
50+ [
51+ "2026-07-15",
52+ 9959.99
53+ ],
54+ [
55+ "2026-07-16",
56+ 9959.99
57+ ],
58+ [
59+ "2026-07-17",
60+ 9959.99
61+ ],
62+ [
63+ "2026-07-18",
64+ 9959.99
65+ ],
66+ [
67+ "2026-07-19",
68+ 9959.99
69+ ],
70+ [
71+ "2026-07-20",
72+ 9959.99
73+ ],
74+ [
75+ "2026-07-21",
76+ 9959.99
77+ ],
78+ [
79+ "2026-07-22",
80+ 9959.99
81+ ],
82+ [
83+ "2026-07-23",
84+ 9959.99
85+ ],
86+ [
87+ "2026-07-24",
88+ 9959.99
89+ ],
90+ [
91+ "2026-07-25",
92+ 9959.99
93+ ],
94+ [
95+ "2026-07-26",
96+ 9959.99
97+ ],
98+ [
99+ "2026-07-27",
100+ 9959.99
101+ ],
102+ [
103+ "2026-07-28",
104+ 9959.99
105+ ],
106+ [
107+ "2026-07-29",
108+ 9959.99
109+ ],
110+ [
111+ "2026-07-30",
112+ 9959.99
113+ ],
114+ [
115+ "2026-07-31",
116+ 9959.99
117+ ],
118+ [
119+ "2026-08-01",
120+ 9959.99
121+ ],
122+ [
123+ "2026-08-02",
124+ 9959.99
125+ ],
126+ [
127+ "2026-08-03",
128+ 9959.99
129+ ],
130+ [
131+ "2026-08-04",
132+ 9959.99
133+ ],
134+ [
135+ "2026-08-05",
136+ 9959.99
137+ ],
138+ [
139+ "2026-08-06",
140+ 9959.99
141+ ],
142+ [
143+ "2026-08-07",
144+ 9959.99
145+ ],
146+ [
147+ "2026-08-08",
148+ 9959.99
149+ ],
150+ [
151+ "2026-08-09",
152+ 9959.99
153+ ],
154+ [
155+ "2026-08-10",
156+ 9959.99
157+ ],
158+ [
159+ "2026-08-11",
160+ 9959.99
161+ ],
162+ [
163+ "2026-08-12",
164+ 9959.99
165+ ],
166+ [
167+ "2026-08-13",
168+ 9959.99
169+ ],
170+ [
171+ "2026-08-14",
172+ 9959.99
173+ ],
174+ [
175+ "2026-08-15",
176+ 9959.99
177+ ],
178+ [
179+ "2026-08-16",
180+ 9959.99
181+ ],
182+ [
183+ "2026-08-17",
184+ 9959.99
185+ ],
186+ [
187+ "2026-08-18",
188+ 9959.99
189+ ],
190+ [
191+ "2026-08-19",
192+ 9959.99
193+ ],
194+ [
195+ "2026-08-20",
196+ 9959.99
197+ ],
198+ [
199+ "2026-08-21",
200+ 9959.99
201+ ],
202+ [
203+ "2026-08-22",
204+ 9959.99
205+ ],
206+ [
207+ "2026-08-23",
208+ 9959.99
209+ ],
210+ [
211+ "2026-08-24",
212+ 9959.99
213+ ],
214+ [
215+ "2026-08-25",
216+ 9959.99
217+ ],
218+ [
219+ "2026-08-26",
220+ 9959.99
221+ ],
222+ [
223+ "2026-08-27",
224+ 9959.99
225+ ],
226+ [
227+ "2026-08-28",
228+ 9959.99
229+ ],
230+ [
231+ "2026-08-29",
232+ 9959.99
233+ ],
234+ [
235+ "2026-08-30",
236+ 9959.99
237+ ],
238+ [
239+ "2026-08-31",
240+ 9959.99
241+ ],
242+ [
243+ "2026-09-01",
244+ 9959.99
245+ ],
246+ [
247+ "2026-09-02",
248+ 9959.99
249+ ],
250+ [
251+ "2026-09-03",
252+ 9959.99
253+ ],
254+ [
255+ "2026-09-04",
256+ 9959.99
257+ ],
258+ [
259+ "2026-09-05",
260+ 9959.99
261+ ],
262+ [
263+ "2026-09-06",
264+ 9959.99
265+ ]
266+ ]
267+ },
268+ "fetchedAt": "2026-09-07T07:21:57.655Z"
269+ },
270+ "expect": {
271+ "minCount": 30,
272+ "kinds": [
273+ "price_observation"
274+ ]
275+ },
276+ "note": "Live capture 2026-09-07 — price_history_component, last 60 daily points",
277+ "capturedAt": "2026-09-07T07:21:57.673Z"
278+}
\ No newline at end of file
279
Menu

Theme
↑↓ navigate ↵ open esc close / or ⌘K anywhere