| 21 | |
21 | |
| 22 | |
22 | |
| 23 | def _con(read_only: bool = True) -> duckdb.DuckDBPyConnection: |
23 | def _con(read_only: bool = True) -> duckdb.DuckDBPyConnection: |
| |
24 | + import resource |
| |
25 | + try: # scans over many Parquet files need file descriptors (macOS default soft limit is 256) |
| |
26 | + soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE) |
| |
27 | + if soft < 65536: |
| |
28 | + resource.setrlimit(resource.RLIMIT_NOFILE, (min(65536, hard), hard)) |
| |
29 | + except (ValueError, OSError): |
| |
30 | + pass |
| 24 | con = duckdb.connect(database=":memory:", config={"threads": 4, "memory_limit": "4GB", "TimeZone": "UTC"}) |
31 | con = duckdb.connect(database=":memory:", config={"threads": 4, "memory_limit": "4GB", "TimeZone": "UTC"}) |
| 25 | return con |
32 | return con |
| 26 | |
33 | |
| 243 | |
250 | |
| 244 | def scoped_view_sql(dataset_id: str, kind: str, scope: dict[str, Any], lake: Lake) -> str | None: |
251 | def scoped_view_sql(dataset_id: str, kind: str, scope: dict[str, Any], lake: Lake) -> str | None: |
| 245 | """Explicit file list from the catalog for symbol-scoped statements (fast path). None → use the glob view.""" |
252 | """Explicit file list from the catalog for symbol-scoped statements (fast path). None → use the glob view.""" |
| 246 | − if not scope["symbols"]: |
253 | + if not scope["symbols"] and not (scope["start"] or scope["end"]): |
| 247 | return None |
254 | return None |
| 248 | with connection() as conn: |
255 | with connection() as conn: |
| 249 | − files = live_files(conn, dataset_id, scope["symbols"], scope["start"], scope["end"]) |
256 | + files = live_files(conn, dataset_id, scope["symbols"] or None, scope["start"], scope["end"]) |
| 250 | if len(files) > 20000: |
257 | if len(files) > 20000: |
| 251 | − return None |
258 | + return None # too broad for an explicit list: directory scan instead |
| 252 | if not files: |
259 | if not files: |
| 253 | return "SELECT * FROM (VALUES (NULL)) t(empty) WHERE false" |
260 | return "SELECT * FROM (VALUES (NULL)) t(empty) WHERE false" |
| 254 | lst = ", ".join("'" + str(lake.resolve(f["path"])).replace("'", "''") + "'" for f in files) |
261 | lst = ", ".join("'" + str(lake.resolve(f["path"])).replace("'", "''") + "'" for f in files) |
| 255 | |
262 | |