Repository Public
1#!/usr/bin/env python32# SP Notes Core3# File: Scripts/spn_recorder.py4# Description: Small recorder used by Scripts/recorders/*.py: calls the SCRATCH backend (never production) and saves each5# real answer as a fixture under Tests/SPNotesAPITests/Fixtures/next/<group>/<name>.json, with an index6# (<group>/_index.json: name, method, path, status, a few headers) that the fixture tests read.7#8# Author: Simon-Pierre Boucher9# Contact: contact@spboucher.ai10#11# Environment: SPN_LIVE_URL (e.g. http://127.0.0.1:18483), SPN_LIVE_TOKEN (dev session token of the scratch database),12# optional SPN_LIVE_ORIGIN (default http://localhost:8483 — the origin the scratch server expects).13import json14import os15import sys16import urllib.error17import urllib.parse18import urllib.request19import uuid2021ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))22FIXTURES = os.path.join(ROOT, "Tests", "SPNotesAPITests", "Fixtures", "next")23KEPT_HEADERS = ("x-existing", "idempotent-replay", "content-type", "content-range", "accept-ranges", "etag", "content-length",24 "x-missing-ids", "x-export-report", "x-bib-url", "content-disposition")252627def new_id():28 return str(uuid.uuid4())293031class Reply:32 def __init__(self, status, headers, body):33 self.status = status34 self.headers = headers35 self.body = body3637 @property38 def json(self):39 return json.loads(self.body.decode("utf-8")) if self.body else None404142class Recorder:43 def __init__(self, group):44 base = os.environ.get("SPN_LIVE_URL", "").rstrip("/")45 token = os.environ.get("SPN_LIVE_TOKEN", "")46 if not base or not token:47 sys.exit("SPN_LIVE_URL and SPN_LIVE_TOKEN are required")48 host = urllib.parse.urlparse(base).hostname or ""49 if host.endswith("spboucher.ai"):50 sys.exit("refusing to record against production")51 self.base = base52 self.token = token53 self.origin = os.environ.get("SPN_LIVE_ORIGIN", "http://localhost:8483")54 self.group = group55 self.dir = os.path.join(FIXTURES, group)56 os.makedirs(self.dir, exist_ok=True)57 self.index = []5859 def call(self, method, path, body=None, save=None, headers=None, raw=None, content_type=None):60 """Sends one request. `save` = fixture name (None = not recorded). Returns a Reply; never raises on 4xx/5xx.61 Extra request `headers` (Range, If-Range…) are kept in the index entry as `requestHeaders`."""62 self._request_headers = dict(headers or {})63 data = raw64 hdrs = {"Cookie": "__Host-note_session=" + self.token, "Accept": "application/json",65 "User-Agent": "SPNotes/2.0.0 (Macintosh; macOS 27.0) native recorder"}66 if method != "GET":67 hdrs["Origin"] = self.origin68 if body is not None:69 data = json.dumps(body).encode("utf-8")70 hdrs["Content-Type"] = "application/json"71 if content_type:72 hdrs["Content-Type"] = content_type73 hdrs.update(headers or {})74 request = urllib.request.Request(self.base + path, data=data, method=method, headers=hdrs)75 try:76 with urllib.request.urlopen(request, timeout=60) as response:77 reply = Reply(response.status, {k.lower(): v for k, v in response.headers.items()}, response.read())78 except urllib.error.HTTPError as error:79 reply = Reply(error.code, {k.lower(): v for k, v in error.headers.items()}, error.read())80 if save:81 self._save(save, method, path, body, reply)82 return reply8384 def _save(self, name, method, path, body, reply):85 content_type = reply.headers.get("content-type", "")86 is_json = "json" in content_type87 # Non-JSON answers: text stays .txt (as before); bytes of a file route (binary types) are kept as .bin.88 is_text = content_type.startswith("text/") or not reply.body89 ext = "json" if is_json else ("txt" if is_text else "bin")90 target = os.path.join(self.dir, name + "." + ext)91 with open(target, "wb") as handle:92 if is_json and reply.body:93 handle.write(json.dumps(json.loads(reply.body.decode("utf-8")), ensure_ascii=False, indent=1, sort_keys=False).encode("utf-8"))94 handle.write(b"\n")95 else:96 handle.write(reply.body)97 self.index.append({"name": name, "file": name + "." + ext, "method": method, "path": path, "status": reply.status,98 "request": body, "headers": {k: v for k, v in reply.headers.items() if k in KEPT_HEADERS}})99 if getattr(self, "_request_headers", None):100 self.index[-1]["requestHeaders"] = self._request_headers101102 def upload(self, path, fields, filename, mime, content, save=None):103 """`POST` multipart/form-data: text `fields` (dict) then one part `file` with the given bytes."""104 boundary = "spn-recorder-" + uuid.uuid4().hex105 parts = []106 for key, value in fields.items():107 parts.append(("--%s\r\nContent-Disposition: form-data; name=\"%s\"\r\n\r\n%s\r\n" % (boundary, key, value)).encode("utf-8"))108 parts.append(("--%s\r\nContent-Disposition: form-data; name=\"file\"; filename=\"%s\"\r\nContent-Type: %s\r\n\r\n"109 % (boundary, filename, mime)).encode("utf-8") + content + b"\r\n")110 parts.append(("--%s--\r\n" % boundary).encode("utf-8"))111 return self.call("POST", path, raw=b"".join(parts), content_type="multipart/form-data; boundary=" + boundary, save=save)112113 def purge_pages(self, ids):114 """Trashes then purges pages (purge needs a session verified < 5 min ago). Returns the ids that could not be purged."""115 left = []116 for page_id in ids:117 self.call("DELETE", "/api/pages/" + page_id)118 if self.call("DELETE", "/api/pages/" + page_id + "?purge=1").status != 200:119 left.append(page_id)120 if left:121 print("%s: %d page(s) left in the trash (purge refused: make a fresh token)" % (self.group, len(left)))122 return left123124 def finish(self):125 with open(os.path.join(self.dir, "_index.json"), "w", encoding="utf-8") as handle:126 json.dump(self.index, handle, ensure_ascii=False, indent=1)127 handle.write("\n")128 print("%s: %d fixtures" % (self.group, len(self.index)))129