Repository Public Research preview
1"""License governance: canonical license codes, permitted operations, tiers.23Open access is NOT open licensing. Every document carries a canonical ``license_code`` with evidence4(the provider field or the URI found in the document) and the operations permitted under the project5policy (docs/data_governance/LICENSE_MATRIX.md). The policy is deliberately conservative: when in6doubt, a document is ingested for metadata only and excluded from training and redistribution.7"""8from __future__ import annotations910import re11from dataclasses import dataclass, field1213# Canonical codes14CC0 = "CC0"15CC_BY = "CC-BY"16CC_BY_SA = "CC-BY-SA"17CC_BY_NC = "CC-BY-NC"18CC_BY_NC_SA = "CC-BY-NC-SA"19CC_BY_ND = "CC-BY-ND"20CC_BY_NC_ND = "CC-BY-NC-ND"21PUBLIC_DOMAIN = "PUBLIC-DOMAIN" # e.g. US Government works, Project Gutenberg public-domain texts22ARXIV_NONEXCLUSIVE = "ARXIV-NONEXCLUSIVE" # arXiv.org perpetual non-exclusive license: no redistribution right23TDM = "TDM" # PMC author manuscripts: text mining only24OTHER_OA = "OTHER-OA" # "open access" without a machine-readable reusable license25UNKNOWN = "UNKNOWN"2627ALL_CODES = [CC0, CC_BY, CC_BY_SA, CC_BY_NC, CC_BY_NC_SA, CC_BY_ND, CC_BY_NC_ND, PUBLIC_DOMAIN, ARXIV_NONEXCLUSIVE, TDM, OTHER_OA, UNKNOWN]2829OPS = ("ingest", "parse", "train", "evaluate", "release_source_text", "release_derivative_text", "commercial_use")303132@dataclass(frozen=True)33class Permission:34 tier: str # A = fully reusable, B = reusable with conditions, C = internal analysis only, X = excluded35 ops: frozenset[str]36 conditions: tuple[str, ...] = ()3738 def allows(self, op: str) -> bool:39 return op in self.ops404142_A = frozenset(OPS)43_B_SA = frozenset(OPS) - {"commercial_use"} | {"commercial_use"} # SA allows commercial; derivatives must be SA44_B_NC = frozenset(OPS) - {"commercial_use"}45_C = frozenset({"ingest", "parse", "evaluate"})46_X = frozenset({"ingest"})4748POLICY: dict[str, Permission] = {49 CC0: Permission("A", _A, ("attribution recommended",)),50 PUBLIC_DOMAIN: Permission("A", _A, ("verify public-domain status per jurisdiction; US works only unless stated",)),51 CC_BY: Permission("A", _A, ("attribution: title, authors, source, license URI, changes stated",)),52 CC_BY_SA: Permission("B", _B_SA, ("derivative text must be released under CC BY-SA", "attribution")),53 CC_BY_NC: Permission("B", _B_NC, ("non-commercial research use only", "derivative text released NC", "attribution")),54 CC_BY_NC_SA: Permission("B", _B_NC, ("non-commercial research use only", "derivative text released CC BY-NC-SA", "attribution")),55 # ND: rewriting is a derivative work; we may analyse but never redistribute derivatives. Training is56 # treated as internal analysis and is DISABLED by default (conservative); flip in config if legal review permits.57 CC_BY_ND: Permission("C", _C, ("no derivative distribution", "training disabled by default")),58 CC_BY_NC_ND: Permission("C", _C, ("no derivative distribution", "non-commercial", "training disabled by default")),59 TDM: Permission("C", _C, ("text-mining only, fair-use principles", "no redistribution")),60 ARXIV_NONEXCLUSIVE: Permission("C", _C, ("arXiv non-exclusive licence grants arXiv, not us, distribution rights", "metadata and internal analysis only")),61 OTHER_OA: Permission("X", _X, ("no machine-readable licence; metadata only",)),62 UNKNOWN: Permission("X", _X, ("licence not determined; metadata only",)),63}6465# Defaults used by the pipeline (overridable in configs/data/*.yaml)66TRAINING_CODES_DEFAULT = (CC0, PUBLIC_DOMAIN, CC_BY, CC_BY_SA, CC_BY_NC, CC_BY_NC_SA)67RELEASE_CODES_DEFAULT = (CC0, PUBLIC_DOMAIN, CC_BY) # the public benchmark subset686970_URI_PATTERNS: list[tuple[re.Pattern, str]] = [71 (re.compile(r"creativecommons\.org/publicdomain/zero", re.I), CC0),72 (re.compile(r"creativecommons\.org/publicdomain/mark", re.I), PUBLIC_DOMAIN),73 (re.compile(r"creativecommons\.org/licenses/by-nc-nd", re.I), CC_BY_NC_ND),74 (re.compile(r"creativecommons\.org/licenses/by-nc-sa", re.I), CC_BY_NC_SA),75 (re.compile(r"creativecommons\.org/licenses/by-nd", re.I), CC_BY_ND),76 (re.compile(r"creativecommons\.org/licenses/by-nc", re.I), CC_BY_NC),77 (re.compile(r"creativecommons\.org/licenses/by-sa", re.I), CC_BY_SA),78 (re.compile(r"creativecommons\.org/licenses/by", re.I), CC_BY),79 (re.compile(r"arxiv\.org/licenses/nonexclusive-distrib", re.I), ARXIV_NONEXCLUSIVE),80 (re.compile(r"gutenberg\.org/license", re.I), PUBLIC_DOMAIN),81]8283_CODE_PATTERNS: list[tuple[re.Pattern, str]] = [84 (re.compile(r"public[\s-]*domain", re.I), PUBLIC_DOMAIN),85 (re.compile(r"^\s*cc0\s*(\d(\.\d)?)?\s*$|^pd$", re.I), CC0),86 (re.compile(r"^\s*cc[\s-]*by[\s-]*nc[\s-]*nd", re.I), CC_BY_NC_ND),87 (re.compile(r"^\s*cc[\s-]*by[\s-]*nc[\s-]*sa", re.I), CC_BY_NC_SA),88 (re.compile(r"^\s*cc[\s-]*by[\s-]*nd", re.I), CC_BY_ND),89 (re.compile(r"^\s*cc[\s-]*by[\s-]*nc", re.I), CC_BY_NC),90 (re.compile(r"^\s*cc[\s-]*by[\s-]*sa", re.I), CC_BY_SA),91 (re.compile(r"^\s*cc[\s-]*by\s*(\d(\.\d)?)?\s*$", re.I), CC_BY),92 (re.compile(r"^\s*tdm\s*$", re.I), TDM),93]949596def normalize_license(code: str | None = None, uri: str | None = None) -> str:97 """Map a provider code and/or a licence URI to a canonical code. URI evidence wins over free-text codes."""98 if uri:99 for pat, canon in _URI_PATTERNS:100 if pat.search(uri):101 return canon102 if code:103 c = code.strip()104 if c.upper() == "PUBLIC-DOMAIN" or c.upper() == "PUBLIC DOMAIN":105 return PUBLIC_DOMAIN106 for pat, canon in _CODE_PATTERNS:107 if pat.search(c):108 return canon109 if c.upper() in ALL_CODES:110 return c.upper()111 if re.search(r"open.?access", c, re.I):112 return OTHER_OA113 return UNKNOWN114115116def permission(code: str) -> Permission:117 return POLICY.get(code, POLICY[UNKNOWN])118119120def permitted_ops(code: str) -> list[str]:121 return sorted(permission(code).ops)122123124def is_training_eligible(code: str, allowed: tuple[str, ...] = TRAINING_CODES_DEFAULT) -> bool:125 return code in allowed and permission(code).allows("train")126127128def is_release_eligible(code: str, allowed: tuple[str, ...] = RELEASE_CODES_DEFAULT) -> bool:129 return code in allowed and permission(code).allows("release_derivative_text")130131132@dataclass133class LicenseRecord:134 code: str135 uri: str | None = None136 evidence: str | None = None # where the licence was read (e.g. "pmc_json.license_code", "jats license xlink:href")137 raw: str | None = None138 tier: str = field(init=False)139 permitted: list[str] = field(init=False)140141 def __post_init__(self) -> None:142 p = permission(self.code)143 self.tier = p.tier144 self.permitted = sorted(p.ops)145146 def to_dict(self) -> dict:147 return {"code": self.code, "uri": self.uri, "evidence": self.evidence, "raw": self.raw, "tier": self.tier, "permitted": self.permitted}148