| 21 |
21 |
|
| 22 |
22 |
|
| 23 |
23 |
def _con(read_only: bool = True) -> duckdb.DuckDBPyConnection: |
|
24 |
+ import resource |
|
25 |
+ try: # scans over many Parquet files need file descriptors (macOS default soft limit is 256) |
|
26 |
+ soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE) |
|
27 |
+ if soft < 65536: |
|
28 |
+ resource.setrlimit(resource.RLIMIT_NOFILE, (min(65536, hard), hard)) |
|
29 |
+ except (ValueError, OSError): |
|
30 |
+ pass |
| 24 |
31 |
con = duckdb.connect(database=":memory:", config={"threads": 4, "memory_limit": "4GB", "TimeZone": "UTC"}) |
| 25 |
32 |
return con |
| 26 |
33 |
|
| 243 |
250 |
|
| 244 |
251 |
def scoped_view_sql(dataset_id: str, kind: str, scope: dict[str, Any], lake: Lake) -> str | None: |
| 245 |
252 |
"""Explicit file list from the catalog for symbol-scoped statements (fast path). None → use the glob view.""" |
| 246 |
|
− if not scope["symbols"]: |
|
253 |
+ if not scope["symbols"] and not (scope["start"] or scope["end"]): |
| 247 |
254 |
return None |
| 248 |
255 |
with connection() as conn: |
| 249 |
|
− files = live_files(conn, dataset_id, scope["symbols"], scope["start"], scope["end"]) |
|
256 |
+ files = live_files(conn, dataset_id, scope["symbols"] or None, scope["start"], scope["end"]) |
| 250 |
257 |
if len(files) > 20000: |
| 251 |
|
− return None |
|
258 |
+ return None # too broad for an explicit list: directory scan instead |
| 252 |
259 |
if not files: |
| 253 |
260 |
return "SELECT * FROM (VALUES (NULL)) t(empty) WHERE false" |
| 254 |
261 |
lst = ", ".join("'" + str(lake.resolve(f["path"])).replace("'", "''") + "'" for f in files) |
| 255 |
262 |
|