limit number of workers
This commit is contained in:
@@ -171,7 +171,7 @@ def extract(
|
|||||||
dedup = DedupIndex(output_dir=cfg.output_dir)
|
dedup = DedupIndex(output_dir=cfg.output_dir)
|
||||||
known_hashes: frozenset = frozenset(dedup._by_hash.keys())
|
known_hashes: frozenset = frozenset(dedup._by_hash.keys())
|
||||||
|
|
||||||
n_workers = workers or os.cpu_count() or 4
|
n_workers = workers or cfg.workers or os.cpu_count() or 4
|
||||||
console.print(f"Using [bold]{n_workers}[/bold] worker processes.")
|
console.print(f"Using [bold]{n_workers}[/bold] worker processes.")
|
||||||
|
|
||||||
owner = {"handle": cfg.owner_handle, "display_name": cfg.owner_display_name}
|
owner = {"handle": cfg.owner_handle, "display_name": cfg.owner_display_name}
|
||||||
|
|||||||
@@ -51,6 +51,7 @@ class ExtractConfig:
|
|||||||
track: TrackConfig = field(default_factory=TrackConfig)
|
track: TrackConfig = field(default_factory=TrackConfig)
|
||||||
classifier: ClassifierConfig = field(default_factory=ClassifierConfig)
|
classifier: ClassifierConfig = field(default_factory=ClassifierConfig)
|
||||||
incremental: bool = True
|
incremental: bool = True
|
||||||
|
workers: Optional[int] = None # None → use CPU count
|
||||||
owner_handle: str = "me"
|
owner_handle: str = "me"
|
||||||
owner_display_name: str = "Me"
|
owner_display_name: str = "Me"
|
||||||
athlete: AthleteConfig | None = None
|
athlete: AthleteConfig | None = None
|
||||||
@@ -109,6 +110,7 @@ def load_config(path: Path) -> ExtractConfig:
|
|||||||
track=track,
|
track=track,
|
||||||
classifier=classifier,
|
classifier=classifier,
|
||||||
incremental=raw.get("incremental", True),
|
incremental=raw.get("incremental", True),
|
||||||
|
workers=raw.get("workers"),
|
||||||
owner_handle=owner.get("handle", "me"),
|
owner_handle=owner.get("handle", "me"),
|
||||||
owner_display_name=owner.get("display_name", "Me"),
|
owner_display_name=owner.get("display_name", "Me"),
|
||||||
athlete=athlete,
|
athlete=athlete,
|
||||||
|
|||||||
@@ -206,6 +206,8 @@ sources:
|
|||||||
|
|
||||||
output:
|
output:
|
||||||
dir: /var/bincio/data
|
dir: /var/bincio/data
|
||||||
|
|
||||||
|
workers: 2 # cap extract parallelism on the VPS (default: all CPUs)
|
||||||
```
|
```
|
||||||
|
|
||||||
Sync and extract (run from your laptop or SSH in):
|
Sync and extract (run from your laptop or SSH in):
|
||||||
|
|||||||
Reference in New Issue
Block a user