Source code for ts_ids_core.mapping_table.convert

import re
from csv import QUOTE_NONE
from io import StringIO
from pathlib import Path
from typing import List

import jsonref
from pandas import DataFrame, concat, read_csv

from ts_ids_core.mapping_table.constants import COMMENT, VISIBLE
from ts_ids_core.mapping_table.locate import get_table_lines
from ts_ids_core.mapping_table.mapping_table import (
    MappingTable,
    construct_body_from_dataframe,
    construct_header_from_columns,
)




class TableError(Exception):
    """Error class for table errors"""





def format_table_lines(table_lines: List[str]) -> List[str]:
    r"""Format the lines of the markdown table

    In markdown tables, any whitespace or a leading/trailing `|` are ignored, so we
    strip these characters from the start and end of each line with a regex.
    """
    new_lines = []
    for line in table_lines:
        line = re.sub(r"^\s*\|?\s*", "", line)
        line = re.sub(r"[\s\|]+\n$", "\n", line)
        new_lines.append(line)
    return new_lines





def markdown_table_lines_to_dataframe(markdown_table_lines: List[str]) -> DataFrame:
    """Convert markdown table lines to a pandas dataframe"""
    formatted_lines = format_table_lines(markdown_table_lines)

    dataframe = read_csv(
        StringIO("".join(formatted_lines)),
        sep="|",
    )
    # Strip all cells of whitespace
    dataframe = dataframe.rename(columns=lambda x: x.strip())
    dataframe = dataframe.apply(
        lambda col: col.map(lambda x: x.strip() if isinstance(x, str) else x)
    )

    dataframe = dataframe.replace("", float("nan"))

    if dataframe.iloc[:, 0].isnull().any():
        raise TableError(
            "Found an empty cell in the first column. The first column is "
            "assumed to contain the IDS paths, which is a required value "
            "in each row."
        )

    return dataframe.drop(0)  # drop the separator row





def readme_lines_to_mapping_table(
    readme_lines: List[str], mapping_table_section_heading: str = "Raw to IDS Mapping"
) -> MappingTable:
    """Extract and convert mapping table lines from readme lines to a MappingTable
    instance.

    Arguments:
        readme_lines -- list of lines from the readme

    Keyword Arguments:
        mapping_table_section_heading -- heading of the mapping table section
    """

    table_lines = get_table_lines(readme_lines, mapping_table_section_heading)
    table_dataframe = markdown_table_lines_to_dataframe(table_lines)

    mapping_table = MappingTable(
        header=construct_header_from_columns(
            column_names=table_dataframe.columns.to_list()
        )
    )
    mapping_table.body = construct_body_from_dataframe(
        keys=mapping_table.keys, dataframe=table_dataframe
    )

    return mapping_table





def readme_to_mapping_table(
    readme_path: Path, mapping_table_section_heading: str = "Raw to IDS Mapping"
) -> MappingTable:
    """Convert a mapping table in a readme file to a MappingTable instance

    Arguments:
        readme_path -- path to the readme file

    Keyword Arguments:
        mapping_table_section_heading -- heading of the mapping table section
    """

    readme_lines = readme_path.read_text(encoding="utf-8").splitlines(keepends=True)
    return readme_lines_to_mapping_table(readme_lines, mapping_table_section_heading)





class UnexpectedJsonError(Exception):
    """Error class for incorrect JSON formats"""





def json_string_to_dict(json_string: str) -> dict:
    """Dereference a json string & ensure it is a dict"""
    contents = jsonref.loads(json_string)
    if not isinstance(contents, dict):
        raise UnexpectedJsonError("Expected the root level to be a dictionary.")
    return contents





def dataframe_to_markdown_lines(table: DataFrame) -> List[str]:
    """Format the mapping table string to our readme conventions.

    pandas.DataFrame.to_markdown() automatically pads each column to be the width of the
    widest cell in the column. This leads to a lot of unnecessary whitespace in many
    rows. To aid readability we use to_csv() with a `|` separator, and then add
    a single space padding to each cell manually.
    """
    table_string = table.to_csv(
        sep="|",
        index=False,
        quoting=QUOTE_NONE,
        lineterminator="\n",
        escapechar="\\",
        # pandas 3.0 started escaping the default quotechar (") too when `escapechar`
        # is set, even with quoting=QUOTE_NONE. We don't want that: only `|` should be
        # escaped. Pointing quotechar at a character that never appears in real data
        # keeps the quote-escaping logic from ever triggering, on any pandas version.
        quotechar="\x01",
    )

    # Add spaces around pipes, but preserve escaped pipes and remove trailing spaces
    table_string = re.sub(r"(?<!\\)\|", " | ", table_string)
    table_string = re.sub(r"\| \n", "|\n", table_string)
    return table_string.splitlines(keepends=True)





def mapping_table_to_markdown_lines(mapping_table: MappingTable) -> List[str]:
    """Convert the MappingTable instance to markdown table lines"""

    table = mapping_table.body_to_dataframe()

    # Only keep records that are visible=True
    table.drop(table[~table[VISIBLE]].index, inplace=True)
    table.drop([VISIBLE, COMMENT], axis=1, errors="ignore", inplace=True)
    # Format the path column
    table["path"] = "`" + table["path"] + "`"

    # Add the separator row to the start of the dataframe
    columns = table.columns.to_list()
    separator = DataFrame([["---"] * len(columns)], columns=columns)
    table_with_separator = concat([separator, table])

    # Rename the columns to use the column_name specified in the header
    table_with_separator.columns = [
        mapping_table.key_to_column_map[column] for column in columns
    ]

    return dataframe_to_markdown_lines(table_with_separator)