Skip to content

ISOBase

Abstract base class for all ISO client implementations. ISOBase uses PEP 695 generics (ISOBase[RawT: StrEnum, ProcessedT: StrEnum]) so that each concrete ISO client is statically typed over its own raw and processed data-type enums.

Responsibilities

  • Defines the five abstract methods that every ISO must implement: download_raw_data, upload_raw_data, download_processed_data, upload_processed_data, and process_raw_data.
  • Provides concrete S3 key-building hooks (_raw_s3_key, _processed_s3_key, _hive_output_path) that produce the standardized hive-partitioned layout.
  • Exposes timezone-aware now() and today() convenience methods.

Hive Path Convention

Processed data is written to:

{dir_name}/year=YYYY/month=MM/day=DD/data.parquet

Raw data follows:

{iso}/{dir_name}/{YYYYMMDD}_{suffix}.{ext}

Classes

ISOBase 

ISOBase(iso: ISO, home_url: str, timezone: str | tzinfo = 'UTC')

Bases: ABC

Source code in src/progridpy/iso/base.py 
12
13
14
15
16
17
18
19
20
21
def __init__(self, iso: ISO, home_url: str, timezone: str | tzinfo = "UTC") -> None:
    self.iso = iso
    self.home_url = home_url
    if isinstance(timezone, str):
        try:
            self.timezone = zoneinfo.ZoneInfo(timezone)
        except zoneinfo.ZoneInfoNotFoundError:
            self.timezone = UTC
    else:
        self.timezone = timezone

Functions

download_raw_data abstractmethod 
download_raw_data(start_date: str | datetime | None = None, end_date: str | datetime | None = None, data_types: RawT | list[RawT] | None = None, download_src: str | FileLocation = ISO, output_dir: str | Path | None = None, overwrite: bool = False, verbose: bool = False) -> list[Path]

Download raw data from S3 or ISO. The start and end dates are inclusive.

Source code in src/progridpy/iso/base.py 
32
33
34
35
36
37
38
39
40
41
42
43
@abstractmethod
def download_raw_data(
    self,
    start_date: str | datetime | None = None,
    end_date: str | datetime | None = None,
    data_types: RawT | list[RawT] | None = None,
    download_src: str | FileLocation = FileLocation.ISO,
    output_dir: str | Path | None = None,
    overwrite: bool = False,
    verbose: bool = False,
) -> list[Path]:
    """Download raw data from S3 or ISO. The start and end dates are inclusive."""
upload_raw_data abstractmethod 
upload_raw_data(start_date: str | datetime | None = None, end_date: str | datetime | None = None, data_types: RawT | list[RawT] | None = None, input_dir: str | Path | None = None, overwrite: bool = False, verbose: bool = False) -> list[str]

Upload raw data files to S3 bucket.

Source code in src/progridpy/iso/base.py 
45
46
47
48
49
50
51
52
53
54
55
@abstractmethod
def upload_raw_data(
    self,
    start_date: str | datetime | None = None,
    end_date: str | datetime | None = None,
    data_types: RawT | list[RawT] | None = None,
    input_dir: str | Path | None = None,
    overwrite: bool = False,
    verbose: bool = False,
) -> list[str]:
    """Upload raw data files to S3 bucket."""
download_processed_data abstractmethod 
download_processed_data(start_date: str | datetime | None = None, end_date: str | datetime | None = None, data_types: ProcessedT | list[ProcessedT] | None = None, output_dir: str | Path | None = None, overwrite: bool = False, verbose: bool = False) -> list[Path]

Download processed data from S3 bucket for the specified date range.

Source code in src/progridpy/iso/base.py 
57
58
59
60
61
62
63
64
65
66
67
@abstractmethod
def download_processed_data(
    self,
    start_date: str | datetime | None = None,
    end_date: str | datetime | None = None,
    data_types: ProcessedT | list[ProcessedT] | None = None,
    output_dir: str | Path | None = None,
    overwrite: bool = False,
    verbose: bool = False,
) -> list[Path]:
    """Download processed data from S3 bucket for the specified date range."""
upload_processed_data abstractmethod 
upload_processed_data(start_date: str | datetime | None = None, end_date: str | datetime | None = None, data_types: ProcessedT | list[ProcessedT] | None = None, input_dir: str | Path | None = None, overwrite: bool = False, verbose: bool = False) -> list[str]

Upload processed data files to S3 bucket.

Source code in src/progridpy/iso/base.py 
69
70
71
72
73
74
75
76
77
78
79
@abstractmethod
def upload_processed_data(
    self,
    start_date: str | datetime | None = None,
    end_date: str | datetime | None = None,
    data_types: ProcessedT | list[ProcessedT] | None = None,
    input_dir: str | Path | None = None,
    overwrite: bool = False,
    verbose: bool = False,
) -> list[str]:
    """Upload processed data files to S3 bucket."""
process_raw_data abstractmethod 
process_raw_data(start_date: str | datetime | None = None, end_date: str | datetime | None = None, data_types: ProcessedT | list[ProcessedT] | None = None, input_dir: str | Path | None = None, output_dir: str | Path | None = None, file_format: Literal['parquet', 'csv'] = 'parquet', overwrite: bool = False, verbose: bool = False) -> list[Path]

Process the downloaded raw data into a standardized format.

Source code in src/progridpy/iso/base.py 
81
82
83
84
85
86
87
88
89
90
91
92
93
@abstractmethod
def process_raw_data(
    self,
    start_date: str | datetime | None = None,
    end_date: str | datetime | None = None,
    data_types: ProcessedT | list[ProcessedT] | None = None,
    input_dir: str | Path | None = None,
    output_dir: str | Path | None = None,
    file_format: Literal["parquet", "csv"] = "parquet",
    overwrite: bool = False,
    verbose: bool = False,
) -> list[Path]:
    """Process the downloaded raw data into a standardized format."""