attributes_csv_loader.cpp

Go to the documentation of this file.

#include "porytiles/infra/services/attributes_csv_loader.hpp"
 
#include <filesystem>
#include <format>
#include <fstream>
#include <optional>
#include <string>
#include <unordered_map>
#include <unordered_set>
#include <vector>
 
#include "porytiles/domain/config/role_pin_definition.hpp"
#include "porytiles/domain/models/layer.hpp"
#include "porytiles/utilities/panic/panic.hpp"
#include "porytiles/utilities/parse_int.hpp"
#include "porytiles/utilities/string_utils.hpp"
#include "porytiles/xcut/config/config_scope_type.hpp"
 
namespace {
 
using namespace porytiles;
 
struct CsvRow {
    std::size_t metatile_id;
    std::vector<std::string> field_cells;        // one trimmed cell per schema value field, in schema order
    std::optional<std::string> layer_type_token; // raw layer_type cell, nullopt when the column or cell is blank
};
 
[[nodiscard]] std::string expected_header_string(const Schema &schema)
{
    std::string header = "id";
    // TODO: this should probably show pins too. Right now, if I use schema.fields(), the pin column prints without the
    // "pin::" prefix. In #350 we'll fix the order dependency, and we should introduce a way to fetch pins from the
    // schema so we can solve the problem here.
    for (const Field &field : schema.value_fields()) {
        header += "," + field.name();
    }
    return header;
}
 
[[nodiscard]] std::optional<std::string>
extract_layer_type_token(const std::vector<std::string> &columns, std::optional<std::size_t> index)
{
    if (!index.has_value() || columns.size() <= index.value()) {
        return std::nullopt;
    }
    std::string token = columns[index.value()];
    trim(token);
    if (token.empty()) {
        return std::nullopt;
    }
    return token;
}
 
ChainableResult<CsvRow> parse_csv_row(
    const std::string &line,
    std::size_t line_index,
    const std::filesystem::path &path,
    const std::vector<std::string> &all_lines,
    const Schema &schema,
    const TextFormatter &format,
    const FileHighlightPrinter &file_printer,
    std::size_t max_columns,
    std::optional<std::size_t> layer_type_token_index)
{
    const std::size_t field_count = schema.value_fields().size();
    auto columns = split(line, ",");
 
    if (columns.size() < 1 + field_count) {
        std::vector<std::string> err_lines{};
        err_lines.push_back(format.format(
            "{}:{}: expected at least {} columns ({}), found {}",
            FormatParam{path.string(), Style::bold},
            FormatParam{line_index + 1, Style::bold},
            FormatParam{1 + field_count},
            FormatParam{expected_header_string(schema)},
            FormatParam{columns.size()}));
        err_lines.emplace_back();
        err_lines.append_range(file_printer.print(all_lines, std::vector{line_index}));
        return FormattableError{std::move(err_lines)};
    }
 
    // The row-level mirror of the header's unexpected-column check: a data row wider than the header shape means the
    // CSV was written for a wider schema, and its extra cells must fail loudly instead of being silently dropped. The
    // header derives @p max_columns as id + value fields + the recognized trailing pin columns.
    if (columns.size() > max_columns) {
        std::vector<std::string> err_lines{};
        err_lines.push_back(format.format(
            "{}:{}: expected at most {} columns, found {}",
            FormatParam{path.string(), Style::bold},
            FormatParam{line_index + 1, Style::bold},
            FormatParam{max_columns},
            FormatParam{columns.size()}));
        err_lines.emplace_back();
        err_lines.append_range(file_printer.print(all_lines, std::vector{line_index}));
        return FormattableError{std::move(err_lines)};
    }
 
    for (std::size_t i = 0; i <= field_count; ++i) {
        trim(columns[i]);
    }
 
    auto id_result = parse_int<int>(columns[0], 0);
    if (!id_result.has_value()) {
        std::vector<std::string> err_lines{};
        err_lines.push_back(format.format(
            "{}:{}: invalid metatile id '{}': {}",
            FormatParam{path.string(), Style::bold},
            FormatParam{line_index + 1, Style::bold},
            FormatParam{columns[0], Style::bold},
            FormatParam{id_result.error()}));
        err_lines.emplace_back();
        err_lines.append_range(file_printer.print(all_lines, std::vector{line_index}));
        return FormattableError{std::move(err_lines)};
    }
 
    if (id_result.value() < 0) {
        std::vector<std::string> err_lines{};
        err_lines.push_back(format.format(
            "{}:{}: metatile id '{}' cannot be negative",
            FormatParam{path.string(), Style::bold},
            FormatParam{line_index + 1, Style::bold},
            FormatParam{columns[0], Style::bold}));
        err_lines.emplace_back();
        err_lines.append_range(file_printer.print(all_lines, std::vector{line_index}));
        return FormattableError{std::move(err_lines)};
    }
 
    std::vector<std::string> field_cells{};
    field_cells.reserve(field_count);
    for (std::size_t i = 0; i < field_count; ++i) {
        field_cells.push_back(columns[1 + i]);
    }
 
    // The active layer_type pin column may sit anywhere in the trailing region (a stale pin column can precede it), so
    // its column index is resolved from the header and passed in rather than assumed adjacent to the fields.
    return CsvRow{
        static_cast<std::size_t>(id_result.value()),
        std::move(field_cells),
        extract_layer_type_token(columns, layer_type_token_index)};
}
 
ChainableResult<std::map<std::size_t, MetatileAttribute>> parse_attributes_csv(
    const std::filesystem::path &path,
    const Schema &schema,
    const ProviderMap &providers,
    const TextFormatter &format,
    const FileHighlightPrinter &file_printer,
    const RolePinDefinitions &role_pins,
    std::map<FieldRole, bool> &active_pin_column_present,
    const UserDiagnostics &diag)
{
    if (!exists(path)) {
        return FormattableError{"{}: file does not exist.", FormatParam{path.string(), Style::bold}};
    }
 
    const std::string expected_header = expected_header_string(schema);
 
    // Slurp entire file into vector for FileHighlightPrinter support
    std::vector<std::string> lines{};
    {
        std::ifstream stream{path};
        std::string line_buf{};
        while (std::getline(stream, line_buf)) {
            std::ignore = trim_line_ending(line_buf);
            lines.push_back(line_buf);
        }
    }
 
    if (lines.empty()) {
        return FormattableError{
            "{}: file is empty, expected header '{}'",
            FormatParam{path.string(), Style::bold},
            FormatParam{expected_header, Style::bold}};
    }
 
    // Cross-check the header line (index 0) against the resolved schema: the columns must be 'id' followed by every
    // schema field name in schema order, then any number of "pin::<role>" pin columns. Anything missing, mis-ordered,
    // or extra is a schema mismatch and fails with a diagnostic naming the column and its position.
    auto header_columns = split(lines[0], ",");
    for (auto &col : header_columns) {
        trim(col);
    }
 
    const std::size_t field_count = schema.value_fields().size();
    auto make_header_error = [&](const std::string &message) -> FormattableError {
        std::vector<std::string> err_lines{};
        err_lines.push_back(message);
        err_lines.emplace_back();
        err_lines.push_back(format.format(
            "Based on resolved attribute schema, expected header: '{}'", FormatParam{expected_header, Style::bold}));
        err_lines.emplace_back();
        err_lines.append_range(file_printer.print(lines, std::vector<std::size_t>{0}));
        return FormattableError{std::move(err_lines)};
    };
 
    for (std::size_t i = 0; i <= field_count; ++i) {
        const std::string &expected_column = i == 0 ? "id" : schema.value_fields()[i - 1].name();
        if (i >= header_columns.size()) {
            return make_header_error(format.format(
                "{}:{}: invalid header: missing column '{}' at position {}",
                FormatParam{path.string(), Style::bold},
                FormatParam{"1", Style::bold},
                FormatParam{expected_column, Style::bold},
                FormatParam{i + 1}));
        }
        if (header_columns[i] != expected_column) {
            return make_header_error(format.format(
                "{}:{}: invalid header: expected column {} to be '{}' but found '{}'",
                FormatParam{path.string(), Style::bold},
                FormatParam{"1", Style::bold},
                FormatParam{i + 1},
                FormatParam{expected_column, Style::bold},
                FormatParam{header_columns[i], Style::bold}));
        }
    }
 
    // Classify the trailing columns after the value fields. A column's kind is read off its name, never its position.
    // In a future update, we'll make it so that the CSV header row is not order dependent. Right now, you can't mix
    // pins together with the regular fields.
    std::optional<std::size_t> layer_type_apply_index;
    std::optional<std::string> ignored_role_pin_column;
    std::unordered_set<std::string> seen_trailing;
    for (std::size_t j = 1 + field_count; j < header_columns.size(); ++j) {
        const std::string &column = header_columns[j];
 
        if (!seen_trailing.insert(column).second) {
            return make_header_error(format.format(
                "{}:{}: invalid header: duplicate column '{}' at position {}",
                FormatParam{path.string(), Style::bold},
                FormatParam{"1", Style::bold},
                FormatParam{column, Style::bold},
                FormatParam{j + 1}));
        }
 
        if (!is_pin_column_name(column)) {
            // Invalid non-pin column present in the "pin region". Again, we should move away from this artificial
            // trailing-pin-region concept. For now, we need to error here.
            return make_header_error(format.format(
                "{}:{}: invalid header: unexpected column '{}' at position {}",
                FormatParam{path.string(), Style::bold},
                FormatParam{"1", Style::bold},
                FormatParam{column, Style::bold},
                FormatParam{j + 1}));
        }
 
        const auto role = role_from_pin_column_name(column);
        if (!role.has_value()) {
            return make_header_error(format.format(
                "{}:{}: invalid header: pin column '{}' at position {} names no known role; the only role is '{}'",
                FormatParam{path.string(), Style::bold},
                FormatParam{"1", Style::bold},
                FormatParam{column, Style::bold},
                FormatParam{j + 1},
                FormatParam{to_string(FieldRole::layer_type), Style::bold}));
        }
 
        if (find_role_pin(role_pins, role.value()) == nullptr) {
            // A well-formed pin column for a role that is not currently pinned: a stale column left behind after the
            // pin was turned off. Its cells are ignored, with a one-time warning.
            if (!ignored_role_pin_column.has_value()) {
                ignored_role_pin_column = column;
            }
            continue;
        }
 
        switch (role.value()) {
        case FieldRole::layer_type:
            layer_type_apply_index = j;
            break;
        }
    }
 
    const std::size_t max_columns = header_columns.size();
 
    // Record, per configured role, whether its active pin column was present in the header. The reader maps this onto
    // the component so the decompiler's round-trip merge knows whether to preserve prior pin state (column present) or
    // pin every row (column absent). A stale ignored column does not count as present.
    for (const RolePinDefinition &pin : role_pins) {
        active_pin_column_present[pin.role] = (pin.role == FieldRole::layer_type) && layer_type_apply_index.has_value();
    }
 
    // A stale pin column is present but not active: ignore its values and say so once for the whole file.
    if (ignored_role_pin_column.has_value()) {
        std::vector<std::string> warn_lines{};
        warn_lines.push_back(format.format(
            "{}: a '{}' column is present but no active role pin uses it. Its values are ignored and the layer type "
            "will be inferred.",
            FormatParam{path.string(), Style::bold},
            FormatParam{ignored_role_pin_column.value(), Style::bold}));
        warn_lines.push_back(format.format(
            "Add a {} entry for the layer_type role to apply the column, or remove the column.",
            FormatParam{"fieldmap.role_pins", Style::bold}));
        diag.warning("role-pin-column", warn_lines);
    }
 
    // Parse data rows (starting at index 1)
    std::map<std::size_t, MetatileAttribute> result{};
    std::unordered_map<std::size_t, std::size_t> id_to_line_index{};
 
    // Applies a filled layer_type cell as an explicit override. The token is only populated from the active pin
    // column, so a nullopt token (blank cell, or no active column) leaves the layer type inferred. A bad token is a
    // hard error with file context.
    auto apply_explicit_layer_type =
        [&](MetatileAttribute &attribute, const CsvRow &row, std::size_t line_index) -> ChainableResult<void> {
        if (!row.layer_type_token.has_value()) {
            return {};
        }
        auto layer_type = layer_type_from_csv_token(row.layer_type_token.value());
        if (!layer_type.has_value()) {
            std::vector<std::string> err_lines{};
            err_lines.push_back(format.format(
                "{}:{}: invalid layer_type '{}'",
                FormatParam{path.string(), Style::bold},
                FormatParam{line_index + 1, Style::bold},
                FormatParam{row.layer_type_token.value(), Style::bold}));
            err_lines.emplace_back();
            err_lines.append_range(file_printer.print(lines, std::vector{line_index}));
            return ChainableResult<void>{FormattableError{std::move(err_lines)}, layer_type};
        }
        attribute.explicit_layer_type(layer_type.value());
        return {};
    };
 
    // Resolves one field cell to its numeric value. A provider-backed field goes through its provider (the ProviderMap
    // membership contract makes a missing provider an internal bug, not a raw fallback); a raw field parses as an
    // unsigned integer capped at the field's maximum, which the binary writer would otherwise silently mask away.
    auto resolve_field_cell =
        [&](const Field &field, const std::string &cell, std::size_t line_index) -> ChainableResult<std::uint32_t> {
        if (field.has_provider()) {
            const auto provider_it = providers.find(field.name());
            if (provider_it == providers.end()) {
                panic(
                    std::format(
                        "parse_attributes_csv: field '{}' has a provider spec but no provider was built for it",
                        field.name()));
            }
            auto lookup_result = provider_it->second->lookup(cell);
            if (!lookup_result.has_value()) {
                std::vector<std::string> err_lines{};
                err_lines.push_back(format.format(
                    "{}:{}: unknown {} '{}'",
                    FormatParam{path.string(), Style::bold},
                    FormatParam{line_index + 1, Style::bold},
                    FormatParam{field.name()},
                    FormatParam{cell, Style::bold}));
                err_lines.emplace_back();
                err_lines.append_range(file_printer.print(lines, std::vector{line_index}));
                return ChainableResult<std::uint32_t>{FormattableError{std::move(err_lines)}, lookup_result};
            }
            return lookup_result.value();
        }
 
        auto int_result = parse_int<long long>(cell, 0);
        if (!int_result.has_value() || int_result.value() < 0) {
            std::vector<std::string> err_lines{};
            err_lines.push_back(format.format(
                "{}:{}: invalid {} value '{}': expected an unsigned integer",
                FormatParam{path.string(), Style::bold},
                FormatParam{line_index + 1, Style::bold},
                FormatParam{field.name()},
                FormatParam{cell, Style::bold}));
            err_lines.emplace_back();
            err_lines.append_range(file_printer.print(lines, std::vector{line_index}));
            return FormattableError{std::move(err_lines)};
        }
        if (int_result.value() > static_cast<long long>(field.max_value())) {
            std::vector<std::string> err_lines{};
            err_lines.push_back(format.format(
                "{}:{}: {} value '{}' exceeds the field's maximum of {}",
                FormatParam{path.string(), Style::bold},
                FormatParam{line_index + 1, Style::bold},
                FormatParam{field.name()},
                FormatParam{cell, Style::bold},
                FormatParam{field.max_value(), Style::bold}));
            err_lines.emplace_back();
            err_lines.append_range(file_printer.print(lines, std::vector{line_index}));
            return FormattableError{std::move(err_lines)};
        }
        return static_cast<std::uint32_t>(int_result.value());
    };
 
    for (std::size_t line_index = 1; line_index < lines.size(); ++line_index) {
        const auto &line = lines[line_index];
 
        if (line.empty()) {
            continue;
        }
 
        ChainableResult<CsvRow> row_result = parse_csv_row(
            line, line_index, path, lines, schema, format, file_printer, max_columns, layer_type_apply_index);
 
        if (!row_result.has_value()) {
            return ChainableResult<std::map<std::size_t, MetatileAttribute>>{
                FormattableError{"Failed to parse CSV row."}, row_result};
        }
 
        const auto &row = row_result.value();
 
        if (id_to_line_index.contains(row.metatile_id)) {
            const std::size_t original_line_index = id_to_line_index.at(row.metatile_id);
            std::vector<std::string> err_lines{};
 
            // Header for duplicate location
            err_lines.push_back(format.format(
                "{}:{}: duplicate metatile id '{}'",
                FormatParam{path.string(), Style::bold},
                FormatParam{line_index + 1, Style::bold},
                FormatParam{row.metatile_id, Style::bold}));
            err_lines.emplace_back();
 
            // File context for duplicate
            err_lines.append_range(file_printer.print(lines, std::vector{line_index}));
            err_lines.emplace_back();
 
            // Note about original location
            err_lines.push_back(format.format(
                "{} originally defined at line {}:",
                FormatParam{"note:", Style::cyan | Style::bold},
                FormatParam{original_line_index + 1}));
 
            // File context for original
            err_lines.append_range(file_printer.print(lines, std::vector{original_line_index}));
 
            return FormattableError{std::move(err_lines)};
        }
        id_to_line_index.emplace(row.metatile_id, line_index);
 
        MetatileAttribute attribute{};
        attribute.layer_type(LayerType::normal);
        for (std::size_t i = 0; i < field_count; ++i) {
            const Field &field = schema.value_fields()[i];
            auto value_result = resolve_field_cell(field, row.field_cells[i], line_index);
            if (!value_result.has_value()) {
                return ChainableResult<std::map<std::size_t, MetatileAttribute>>{
                    FormattableError{"Failed to resolve CSV field cell."}, value_result};
            }
            attribute.field(field.name(), value_result.value());
        }
        if (const auto applied = apply_explicit_layer_type(attribute, row, line_index); !applied.has_value()) {
            return ChainableResult<std::map<std::size_t, MetatileAttribute>>{
                FormattableError{"Failed to apply layer_type override."}, applied};
        }
        result.emplace(row.metatile_id, std::move(attribute));
    }
 
    return result;
}
 
} // namespace
 
namespace porytiles {
 
ChainableResult<AttributesCsvLoadResult> AttributesCsvLoader::load(
    const std::filesystem::path &path,
    const Schema &schema,
    const ProviderMap &providers,
    const std::string &tileset_name) const
{
    // Resolve the role pins under the file's owning tileset scope. When compiling a secondary, the paired primary's CSV
    // loads through this same loader with the primary's name, so its config resolves under the primary's scope. (The
    // result type has a comma, so it cannot go through the PT_TRY_ASSIGN macros; unwrap by hand.)
    auto role_pins_cv = config_->role_pins(ConfigScopeType::tileset, tileset_name);
    if (!role_pins_cv.has_value()) {
        return ChainableResult<AttributesCsvLoadResult>{FormattableError{"Failed to resolve role_pins."}, role_pins_cv};
    }
    const RolePinDefinitions &role_pins = role_pins_cv.value();
 
    std::map<FieldRole, bool> active_pin_column_present;
    auto parsed = parse_attributes_csv(
        path, schema, providers, *format_, *file_printer_, role_pins, active_pin_column_present, *diag_);
    if (!parsed.has_value()) {
        return ChainableResult<AttributesCsvLoadResult>{FormattableError{"Failed to load attributes CSV."}, parsed};
    }
 
    return AttributesCsvLoadResult{std::move(parsed).value(), std::move(active_pin_column_present)};
}
 
} // namespace porytiles