Skip to content

Dataframe extractor

gridgulp.extractors.dataframe_extractor

Extract DataFrames from detected table regions with header detection.

HeaderDetectionResult

Bases: BaseModel

Result of header detection analysis.

DataFrameExtractor 

DataFrameExtractor(min_data_rows: int = 2, min_data_density: float = 0.3)

Extract pandas DataFrames from sheet data with intelligent header detection.

Initialize extractor.

Parameters:

  • min_data_rows (int, default: 2 ) –

    Minimum number of data rows (excluding headers) for valid table

  • min_data_density (float, default: 0.3 ) –

    Minimum ratio of non-empty cells for valid table

Source code in src/gridgulp/extractors/dataframe_extractor.py 
42
43
44
45
46
47
48
49
50
def __init__(self, min_data_rows: int = 2, min_data_density: float = 0.3):
    """Initialize extractor.

    Args:
        min_data_rows: Minimum number of data rows (excluding headers) for valid table
        min_data_density: Minimum ratio of non-empty cells for valid table
    """
    self.min_data_rows = min_data_rows
    self.min_data_density = min_data_density

extract_dataframe 

extract_dataframe(sheet_data: SheetData, cell_range: CellRange, detect_headers: bool = True) -> tuple[pd.DataFrame | None, HeaderDetectionResult | None, float]

Extract a DataFrame from the specified range.

Parameters:

  • sheet_data (SheetData) –

    Sheet containing the data

  • cell_range (CellRange) –

    Range to extract

  • detect_headers (bool, default: True ) –

    Whether to detect headers automatically

Returns:

  • DataFrame | None –

    Tuple of (dataframe, header_info, quality_score)

  • HeaderDetectionResult | None –

    Returns (None, None, 0.0) if extraction fails

Source code in src/gridgulp/extractors/dataframe_extractor.py 
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
def extract_dataframe(
    self, sheet_data: SheetData, cell_range: CellRange, detect_headers: bool = True
) -> tuple[pd.DataFrame | None, HeaderDetectionResult | None, float]:
    """Extract a DataFrame from the specified range.

    Args:
        sheet_data: Sheet containing the data
        cell_range: Range to extract
        detect_headers: Whether to detect headers automatically

    Returns:
        Tuple of (dataframe, header_info, quality_score)
        Returns (None, None, 0.0) if extraction fails
    """
    # Get raw cell data
    range_data = sheet_data.get_range_data(
        cell_range.start_row,
        cell_range.start_col,
        cell_range.end_row,
        cell_range.end_col,
    )

    if not range_data:
        return None, None, 0.0

    # Convert to values matrix
    values_matrix = self._extract_values_matrix(range_data)

    if values_matrix is None or len(values_matrix) == 0:
        return None, None, 0.0

    # Check data density
    density = self._calculate_density(values_matrix)
    logger.debug(f"Data density: {density:.2f}, min required: {self.min_data_density}")
    if density < self.min_data_density:
        logger.debug(f"Low data density: {density:.2f} < {self.min_data_density}")
        return None, None, 0.0

    # Detect headers if requested
    header_info = None
    logger.debug(f"Detect headers: {detect_headers}")
    if detect_headers:
        header_info = self._detect_headers(values_matrix, sheet_data, cell_range)
        logger.debug(f"Header info result: {header_info}")

    # Extract DataFrame based on header detection
    df = self._create_dataframe(values_matrix, header_info)

    if df is None:
        # Still return header info even if we can't create a valid dataframe
        return None, header_info, 0.0

    if len(df) < self.min_data_rows:
        # For plate formats, we might have valid data even with few rows
        if header_info and header_info.table_type == "plate_map":
            # Plate maps are valid even with minimal data
            quality_score = 0.95
            return df, header_info, quality_score
        else:
            return None, header_info, 0.0

    # Calculate quality score
    quality_score = self._calculate_quality_score(df, header_info, density)

    return df, header_info, quality_score