Skip to content

dumper

This file is meant to help dump information to files

Dumper

The dumper class is intended to help write information to files. Currently, the supported dump file types are: csv or json.

Attributes:

Name Type Description
file_name str

The name of the file to write data to.
file_type str

The type of the file (either "csv" or "json") determined from the file name.

Methods:

Name Description
write

Writes information to the specified output file based on the file type.
_csv_write

Writes information to a CSV file.
_json_write

Writes information to a JSON file.
Example

CSV usage: 

dumper = Dumper("populations.csv")
# Eugene, OR has a population of 175096
# Livermore, CA has a population of 86803
population_data = {
    "City": ["Eugene", "Livermore"],
    "State": ["OR", "CA"],
    "Population": [175096, 86803]
}
dumper.write(population_data, "w")  # Output will be written to populations.csv

Example

JSON usage: 

dumper = Dumper("populations.json")
population_data = {
    "OR": {"Eugene": 175096, "Portland": 641162},
    "CA": {"Livermore": 86803, "San Francisco": 815201}
}
dumper.write(population_data, "w")  # Output will be written to populations.json

Source code in merlin/common/dumper.py 
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
class Dumper:  # pylint: disable=R0903
    """
    The dumper class is intended to help write information to files.
    Currently, the supported dump file types are: csv or json.

    Attributes:
        file_name (str): The name of the file to write data to.
        file_type (str): The type of the file (either "csv" or "json") determined
            from the file name.

    Methods:
        write: Writes information to the specified output file based on the file type.
        _csv_write: Writes information to a CSV file.
        _json_write: Writes information to a JSON file.

    Example:
        CSV usage:
        ```python
        dumper = Dumper("populations.csv")
        # Eugene, OR has a population of 175096
        # Livermore, CA has a population of 86803
        population_data = {
            "City": ["Eugene", "Livermore"],
            "State": ["OR", "CA"],
            "Population": [175096, 86803]
        }
        dumper.write(population_data, "w")  # Output will be written to populations.csv
        ```

    Example:
        JSON usage:
        ```python
        dumper = Dumper("populations.json")
        population_data = {
            "OR": {"Eugene": 175096, "Portland": 641162},
            "CA": {"Livermore": 86803, "San Francisco": 815201}
        }
        dumper.write(population_data, "w")  # Output will be written to populations.json
        ```
    """

    def __init__(self, file_name: str):
        """
        Initializes the Dumper class and validates the file type.

        Args:
            file_name: The name of the file to write data to.

        Raises:
            ValueError: If the file type is not supported. Supported types are CSV and JSON.
        """
        supported_types = ["csv", "json"]

        valid_file = False
        for stype in supported_types:
            if file_name.endswith(stype):
                valid_file = True
                self.file_type = stype

        if not valid_file:
            raise ValueError(f"Invalid file type for {file_name}. Supported file types are: {supported_types}.")

        self.file_name = file_name

    def write(self, info_to_write: Dict, fmode: str):
        """
        Writes information to the specified output file.

        This method determines the file type and calls the appropriate
        method to write the data.

        Args:
            info_to_write: The information to write to the output file.
            fmode: The file write mode ("w" for write, "a" for append, etc.).
        """
        if self.file_type == "csv":
            self._csv_write(info_to_write, fmode)
        elif self.file_type == "json":
            self._json_write(info_to_write, fmode)

    def _csv_write(self, csv_to_dump: Dict[str, List], fmode: str):
        """
        Writes information to a CSV file.

        Args:
            csv_to_dump: The data to write to the CSV file. Keys are column
                headers and values are column values.
            fmode: The file write mode ("w" for write, "a" for append, etc.).
        """
        # If we have statuses to write, create a csv writer object and write to the csv file
        with open(self.file_name, fmode) as outfile:
            csv_writer = csv.writer(outfile)
            if fmode == "w":
                csv_writer.writerow(csv_to_dump.keys())
            csv_writer.writerows(zip(*csv_to_dump.values()))

    def _json_write(self, json_to_dump: Dict[str, Dict], fmode: str):
        """
        Writes information to a JSON file.

        Args:
            json_to_dump: The data to write to the JSON file.
            fmode: The file write mode ("w" for write, "a" for append, etc.).
        """
        # Appending to json requires file mode to be r+ for json.load
        if fmode == "a":
            fmode = "r+"

        with open(self.file_name, fmode) as outfile:
            # If we're appending, read in the existing file data
            if fmode == "r+":
                file_data = json.load(outfile)
                json_to_dump.update(file_data)
                outfile.seek(0)
            # Write to the outfile
            json.dump(json_to_dump, outfile)

__init__(file_name)

Initializes the Dumper class and validates the file type.

Parameters:

Name Type Description Default
file_name str

The name of the file to write data to.

 required

Raises:

Type Description
ValueError

If the file type is not supported. Supported types are CSV and JSON.
Source code in merlin/common/dumper.py 
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
def __init__(self, file_name: str):
    """
    Initializes the Dumper class and validates the file type.

    Args:
        file_name: The name of the file to write data to.

    Raises:
        ValueError: If the file type is not supported. Supported types are CSV and JSON.
    """
    supported_types = ["csv", "json"]

    valid_file = False
    for stype in supported_types:
        if file_name.endswith(stype):
            valid_file = True
            self.file_type = stype

    if not valid_file:
        raise ValueError(f"Invalid file type for {file_name}. Supported file types are: {supported_types}.")

    self.file_name = file_name

write(info_to_write, fmode)

Writes information to the specified output file.

This method determines the file type and calls the appropriate method to write the data.

Parameters:

Name Type Description Default
info_to_write Dict

The information to write to the output file.

 required
fmode str

The file write mode ("w" for write, "a" for append, etc.).

 required
Source code in merlin/common/dumper.py 
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
def write(self, info_to_write: Dict, fmode: str):
    """
    Writes information to the specified output file.

    This method determines the file type and calls the appropriate
    method to write the data.

    Args:
        info_to_write: The information to write to the output file.
        fmode: The file write mode ("w" for write, "a" for append, etc.).
    """
    if self.file_type == "csv":
        self._csv_write(info_to_write, fmode)
    elif self.file_type == "json":
        self._json_write(info_to_write, fmode)

dump_handler(dump_file, dump_info)

Handles the process of creating a Dumper object and writing data to an output file.

This function checks if the specified dump file exists to determine the appropriate file write mode (append or write). It then creates a Dumper object and writes the provided information to the file, logging the process.

Parameters:

Name Type Description Default
dump_file str

The filepath to the file where data will be dumped.

 required
dump_info Dict

A dictionary containing the information to be written to the dump_file.

 required
Source code in merlin/common/dumper.py 
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
def dump_handler(dump_file: str, dump_info: Dict):
    """
    Handles the process of creating a Dumper object and writing data
    to an output file.

    This function checks if the specified dump file exists to determine
    the appropriate file write mode (append or write). It then creates
    a Dumper object and writes the provided information to the file,
    logging the process.

    Args:
        dump_file: The filepath to the file where data will be dumped.
        dump_info: A dictionary containing the information to be written
            to the `dump_file`.
    """
    # Create a dumper object to help us write to dump_file
    dumper = Dumper(dump_file)

    # Get the correct file write mode and log message
    fmode = "a" if os.path.exists(dump_file) else "w"
    write_type = "Writing" if fmode == "w" else "Appending"
    LOG.info(f"{write_type} to {dump_file}...")

    # Write the output
    dumper.write(dump_info, fmode)
    LOG.info(f"{write_type} complete.")