Logging

import logging
import os

LOG_FORMAT = "%(asctime)s - %(name)s - %(levelname)s - %(filename)s:%(lineno)d - %(message)s"

def setup_logging(log_file=None):
    logger = logging.getLogger()

    # Define the log level
    logger.setLevel(logging.DEBUG)

    # Create a formatter
    formatter = logging.Formatter(LOG_FORMAT)

    # Create a console handler and set the formatter
    console_handler = logging.StreamHandler()
    console_handler.setFormatter(formatter)
    logger.addHandler(console_handler)

    # Optionally, create a file handler and set the formatter
    if log_file:
        file_handler = logging.FileHandler(log_file)
        file_handler.setFormatter(formatter)
        logger.addHandler(file_handler)

    return logger

logging.basicConfig(filename="preprocess.log", level=logging.INFO, format="[%(asctime)s] [%(levelname)s] %(message)s", datefmt="%Y-%m-%d %H:%M:%S")