splits.py 594 B

123456789101112131415161718192021
  1. from __future__ import annotations
  2. import json
  3. from pathlib import Path
  4. def load_id_txt(path: str | Path) -> list[str]:
  5. path = Path(path)
  6. return [line.strip() for line in path.read_text(encoding="utf-8", errors="ignore").splitlines() if line.strip()]
  7. def load_json_split(path: str | Path) -> dict[str, list[str]]:
  8. path = Path(path)
  9. obj = json.loads(path.read_text(encoding="utf-8"))
  10. result: dict[str, list[str]] = {}
  11. for key, value in obj.items():
  12. result[key] = [str(item) for item in value]
  13. return result
  14. __all__ = ["load_id_txt", "load_json_split"]