Repository Public Research preview
1"""Versioned data schema (pydantic v2) for documents, passages, rewrites, pairs and splits.23All identifiers are deterministic and immutable (see ``htr.utils.hashing.short_id``). Offsets refer to the4normalised document text stored next to the parsed record so that every passage is traceable to its5source bytes.6"""7from __future__ import annotations89from typing import Literal, Optional1011from pydantic import BaseModel, Field1213SCHEMA_VERSION = "htr-data-1.0"1415SourceName = Literal["pmc", "arxiv", "crossref", "bookshelf", "openstax", "gutenberg", "local", "mock"]16SectionRole = Literal["abstract", "introduction", "background", "methods", "results", "discussion", "conclusion", "related_work", "theory", "data", "limitations", "other", "caption", "chapter", "preface"]17DocType = Literal["article", "review", "book_chapter", "book", "preprint", "lecture_notes", "other"]181920class LicenseInfo(BaseModel):21 code: str22 uri: Optional[str] = None23 evidence: Optional[str] = None24 raw: Optional[str] = None25 tier: str26 permitted: list[str]272829class Document(BaseModel):30 schema_version: str = SCHEMA_VERSION31 doc_id: str32 source: SourceName33 source_ids: dict[str, str] = Field(default_factory=dict) # pmcid, pmid, doi, arxiv_id, isbn, gutenberg_id...34 title: Optional[str] = None35 authors: list[str] = Field(default_factory=list)36 author_group: Optional[str] = None # normalised first-author surname + initial, used for author-safe splits37 venue: Optional[str] = None38 publisher: Optional[str] = None39 doc_type: DocType = "article"40 pub_date: Optional[str] = None # ISO date (YYYY, YYYY-MM or YYYY-MM-DD)41 pub_year: Optional[int] = None42 pub_date_evidence: Optional[str] = None # e.g. "jats pub-date[pub-type=ppub]", "arxiv v1 submitted"43 pre2018_verified: bool = False44 subject_area: Optional[str] = None # coarse discipline label (see htr.data.taxonomy)45 subject_raw: list[str] = Field(default_factory=list)46 language: Optional[str] = None47 source_url: str48 license: LicenseInfo49 acquired_at: str50 raw_path: Optional[str] = None51 raw_sha256: Optional[str] = None52 raw_bytes: Optional[int] = None53 raw_format: Literal["jats", "bits", "latex", "pdf", "txt", "html", "epub", "json"] = "jats"54 provider_md5: Optional[str] = None55 provider_md5_ok: Optional[bool] = None56 retracted: Optional[bool] = None57 notes: list[str] = Field(default_factory=list)585960class Passage(BaseModel):61 schema_version: str = SCHEMA_VERSION62 passage_id: str63 doc_id: str64 section_id: str65 section_title: Optional[str] = None66 section_role: SectionRole = "other"67 section_depth: int = 168 para_index: int # index of the paragraph within the document body69 text: str # normalised paragraph text with protected spans kept verbatim70 char_start: int # offsets into the parsed document text file71 char_end: int72 n_words: int73 n_sentences: int74 language: Optional[str] = None75 quality: float = 0.076 flags: list[str] = Field(default_factory=list)77 protected: dict[str, list[str]] = Field(default_factory=dict) # citations / math / numbers / urls found78 text_sha256: str79 eligible: bool = False808182class RewriteRequest(BaseModel):83 request_id: str84 passage_id: str85 variant: str # A..E86 sample_index: int = 087 provider: str88 model: str89 prompt_version: str90 params: dict = Field(default_factory=dict)919293class Rewrite(BaseModel):94 schema_version: str = SCHEMA_VERSION95 rewrite_id: str96 request_id: str97 passage_id: str98 doc_id: str99 variant: str100 sample_index: int = 0101 provider: str102 model: str103 prompt_version: str104 text: Optional[str] = None105 status: Literal["ok", "refused", "error", "empty", "truncated"] = "ok"106 error: Optional[str] = None107 provider_request_id: Optional[str] = None108 usage: dict = Field(default_factory=dict)109 cost_usd: Optional[float] = None110 stop_reason: Optional[str] = None111 created_at: str112 latency_s: Optional[float] = None113 cached: bool = False114 batch_id: Optional[str] = None115 protect_version: str = "1.0" # version of the placeholder protocol applied to the passage sent to the LLM116 prompt_input_sha256: Optional[str] = None # hash of the exact (protected) passage text sent117118119class PairValidation(BaseModel):120 passed: bool121 checks: dict[str, bool]122 scores: dict[str, float]123 reasons: list[str] = Field(default_factory=list)124 validator_version: str125126127class Pair(BaseModel):128 schema_version: str = SCHEMA_VERSION129 pair_id: str130 passage_id: str131 doc_id: str132 rewrite_id: str133 variant: str134 sample_index: int135 source_text: str # the LLM rewrite (model INPUT)136 target_text: str # the original human passage (model TARGET)137 metadata: dict = Field(default_factory=dict) # discipline, pub_year, era, section_role, source, license138 validation: Optional[PairValidation] = None139 quarantined: bool = False140141142class SplitAssignment(BaseModel):143 doc_id: str144 split: Literal["train", "val", "test", "heldout_domain", "heldout_period", "heldout_author", "quarantine"]145 group_keys: dict[str, str] = Field(default_factory=dict)146