fix: Upload CSV as Dataset (#34763)

tests/unit_tests/commands/databases/csv_reader_test.py

@@ -421,6 +421,561 @@ def test_csv_reader_file_metadata_invalid_file():
     )
 
 
+def test_csv_reader_integer_in_float_column():
+    csv_data = [
+        ["Name", "Score", "City"],
+        ["name1", 25.5, "city1"],
+        ["name2", 25, "city2"],
+    ]
+
+    csv_reader = CSVReader(
+        options=CSVReaderOptions(column_data_types={"Score": "float"})
+    )
+
+    df = csv_reader.file_to_dataframe(create_csv_file(csv_data))
+
+    assert df.shape == (2, 3)
+    assert df["Score"].dtype == "float64"
+
+
+def test_csv_reader_object_type_auto_inferring():
+    # this case below won't raise a error
+    csv_data = [
+        ["Name", "id", "City"],
+        ["name1", 25.5, "city1"],
+        ["name2", 15, "city2"],
+        ["name3", 123456789086, "city3"],
+        ["name4", "abc", "city4"],
+        ["name5", 4.75, "city5"],
+    ]
+
+    csv_reader = CSVReader()
+
+    df = csv_reader.file_to_dataframe(create_csv_file(csv_data))
+
+    assert df.shape == (5, 3)
+    # pandas automatically infers the type if column_data_types is not informed
+    # if there's only one string in the column it converts the whole column to object
+    assert df["id"].dtype == "object"
+
+
+def test_csv_reader_float_type_auto_inferring():
+    csv_data = [
+        ["Name", "id", "City"],
+        ["name1", "25", "city1"],
+        ["name2", "15", "city2"],
+        ["name3", "123456789086", "city3"],
+        ["name5", "4.75", "city5"],
+    ]
+
+    csv_reader = CSVReader()
+
+    df = csv_reader.file_to_dataframe(create_csv_file(csv_data))
+
+    assert df.shape == (4, 3)
+    # The type here is automatically inferred to float due to 4.75 value
+    assert df["id"].dtype == "float64"
+
+
+def test_csv_reader_int_type_auto_inferring():
+    csv_data = [
+        ["Name", "id", "City"],
+        ["name1", "0", "city1"],
+        ["name2", "15", "city2"],
+        ["name3", "123456789086", "city3"],
+        ["name5", "45", "city5"],
+    ]
+
+    csv_reader = CSVReader()
+
+    df = csv_reader.file_to_dataframe(create_csv_file(csv_data))
+
+    assert df.shape == (4, 3)
+    assert df["id"].dtype == "int64"
+
+
+def test_csv_reader_bigint_type_auto_inferring():
+    csv_data = [
+        ["Name", "id", "City"],
+        ["name1", "9223372036854775807", "city1"],
+        ["name2", "9223372036854775806", "city2"],
+        ["name3", "1234567890123456789", "city3"],
+        ["name4", "0", "city4"],
+        ["name5", "-9223372036854775808", "city5"],
+    ]
+
+    csv_reader = CSVReader()
+
+    df = csv_reader.file_to_dataframe(create_csv_file(csv_data))
+
+    assert df.shape == (5, 3)
+    assert df["id"].dtype == "int64"
+    assert df.iloc[0]["id"] == 9223372036854775807
+    assert df.iloc[4]["id"] == -9223372036854775808
+
+
+def test_csv_reader_int_typing():
+    csv_data = [
+        ["Name", "id", "City"],
+        ["name1", "0", "city1"],
+        ["name2", "15", "city2"],
+        ["name3", "123456789086", "city3"],
+        ["name5", "45", "city5"],
+    ]
+
+    csv_reader = CSVReader(options=CSVReaderOptions(column_data_types={"id": "int"}))
+
+    df = csv_reader.file_to_dataframe(create_csv_file(csv_data))
+
+    assert df.shape == (4, 3)
+    assert df["id"].dtype == "int64"
+
+
+def test_csv_reader_float_typing():
+    csv_data = [
+        ["Name", "score", "City"],
+        ["name1", "0", "city1"],
+        ["name2", "15.3", "city2"],
+        ["name3", "45", "city3"],
+        ["name5", "23.1342", "city5"],
+    ]
+
+    csv_reader = CSVReader(
+        options=CSVReaderOptions(column_data_types={"score": "float"})
+    )
+
+    df = csv_reader.file_to_dataframe(create_csv_file(csv_data))
+
+    assert df.shape == (4, 3)
+    assert df["score"].dtype == "float64"
+
+
+def test_csv_reader_multiple_errors_display():
+    """Test that multiple errors are displayed with proper formatting."""
+    csv_data = [
+        ["Name", "Age", "Score"],
+        ["Alice", "25", "95.5"],
+        ["Bob", "invalid1", "87.2"],
+        ["Charlie", "invalid2", "92.1"],
+        ["Diana", "invalid3", "88.5"],
+        ["Eve", "invalid4", "90.0"],
+        ["Frank", "30", "85.5"],
+    ]
+
+    csv_reader = CSVReader(options=CSVReaderOptions(column_data_types={"Age": "int64"}))
+
+    with pytest.raises(DatabaseUploadFailed) as ex:
+        csv_reader.file_to_dataframe(create_csv_file(csv_data))
+
+    error_msg = str(ex.value)
+    assert "Cannot convert column 'Age' to int64" in error_msg
+    assert "Found 4 error(s):" in error_msg
+    assert "Line 3: 'invalid1' cannot be converted to int64" in error_msg
+    assert "Line 4: 'invalid2' cannot be converted to int64" in error_msg
+    assert "Line 5: 'invalid3' cannot be converted to int64" in error_msg
+    assert "Line 6: 'invalid4' cannot be converted to int64" in error_msg
+    # With MAX_DISPLAYED_ERRORS = 5, all 4 errors should be shown without truncation
+    assert "and" not in error_msg or "more error(s)" not in error_msg
+
+
+def test_csv_reader_non_numeric_in_integer_column():
+    csv_data = [
+        ["Name", "Age", "City"],
+        ["name1", "abc", "city1"],
+        ["name2", "25", "city2"],
+    ]
+
+    csv_reader = CSVReader(options=CSVReaderOptions(column_data_types={"Age": "int64"}))
+
+    with pytest.raises(DatabaseUploadFailed) as ex:
+        csv_reader.file_to_dataframe(create_csv_file(csv_data))
+
+    error_msg = str(ex.value)
+    assert "Cannot convert column 'Age' to int64" in error_msg
+    assert "Found 1 error(s):" in error_msg
+    assert "Line 2: 'abc' cannot be converted to int64" in error_msg
+
+
+def test_csv_reader_non_numeric_in_float_column():
+    csv_data = [
+        ["Name", "Score", "City"],
+        ["name1", "5.3", "city1"],
+        ["name2", "25.5", "city2"],
+        ["name3", "24.5", "city3"],
+        ["name4", "1.0", "city4"],
+        ["name5", "one point five", "city5"],
+    ]
+
+    csv_reader = CSVReader(
+        options=CSVReaderOptions(column_data_types={"Score": "float64"})
+    )
+
+    with pytest.raises(DatabaseUploadFailed) as ex:
+        csv_reader.file_to_dataframe(create_csv_file(csv_data))
+
+    error_msg = str(ex.value)
+    assert "Cannot convert column 'Score' to float64" in error_msg
+    assert "Found 1 error(s):" in error_msg
+    assert "Line 6: 'one point five' cannot be converted to float64" in error_msg
+
+
+def test_csv_reader_improved_error_detection_int32():
+    """Test improved error detection for int32 type casting."""
+    csv_data = [
+        ["Name", "ID", "City"],
+        ["name1", "123", "city1"],
+        ["name2", "456", "city2"],
+        ["name3", "not_a_number", "city3"],
+        ["name4", "789", "city4"],
+    ]
+
+    csv_reader = CSVReader(options=CSVReaderOptions(column_data_types={"ID": "int32"}))
+
+    with pytest.raises(DatabaseUploadFailed) as ex:
+        csv_reader.file_to_dataframe(create_csv_file(csv_data))
+
+    error_msg = str(ex.value)
+    assert "Cannot convert column 'ID' to int32" in error_msg
+    assert "Found 1 error(s):" in error_msg
+    assert "Line 4: 'not_a_number' cannot be converted to int32" in error_msg
+
+
+def test_csv_reader_improved_error_detection_float32():
+    """Test improved error detection for float32 type casting."""
+    csv_data = [
+        ["Name", "Score", "City"],
+        ["name1", "1.5", "city1"],
+        ["name2", "2.7", "city2"],
+        ["name3", "invalid_float", "city3"],
+    ]
+
+    csv_reader = CSVReader(
+        options=CSVReaderOptions(column_data_types={"Score": "float32"})
+    )
+
+    with pytest.raises(DatabaseUploadFailed) as ex:
+        csv_reader.file_to_dataframe(create_csv_file(csv_data))
+
+    error_msg = str(ex.value)
+    assert "Cannot convert column 'Score' to float32" in error_msg
+    assert "Found 1 error(s):" in error_msg
+    assert "Line 4: 'invalid_float' cannot be converted to float32" in error_msg
+
+
+def test_csv_reader_error_detection_with_header_row():
+    """Test that line numbers are correctly calculated with custom header row."""
+    csv_data = [
+        ["skip_this_row", "skip", "skip"],
+        ["Name", "Age", "City"],
+        ["name1", "25", "city1"],
+        ["name2", "invalid_age", "city2"],
+    ]
+
+    csv_reader = CSVReader(
+        options=CSVReaderOptions(header_row=1, column_data_types={"Age": "int"})
+    )
+
+    with pytest.raises(DatabaseUploadFailed) as ex:
+        csv_reader.file_to_dataframe(create_csv_file(csv_data))
+
+    error_msg = str(ex.value)
+    assert "Cannot convert column 'Age' to int" in error_msg
+    assert "Found 1 error(s):" in error_msg
+    assert "Line 4: 'invalid_age' cannot be converted to int" in error_msg
+
+
+def test_csv_reader_error_detection_first_row_error():
+    """Test error detection when the first data row has the error."""
+
+    csv_data = [
+        ["Name", "Age", "City"],
+        ["name1", "not_a_number", "city1"],
+        ["name2", "25", "city2"],
+    ]
+
+    csv_reader = CSVReader(options=CSVReaderOptions(column_data_types={"Age": "int64"}))
+
+    with pytest.raises(DatabaseUploadFailed) as ex:
+        csv_reader.file_to_dataframe(create_csv_file(csv_data))
+
+    error_msg = str(ex.value)
+    assert "Cannot convert column 'Age' to int64" in error_msg
+    assert "Found 1 error(s):" in error_msg
+    assert "Line 2: 'not_a_number' cannot be converted to int64" in error_msg
+
+
+def test_csv_reader_error_detection_missing_column():
+    """Test that missing columns are handled gracefully."""
+    csv_data = [
+        ["Name", "City"],
+        ["name1", "city1"],
+        ["name2", "city2"],
+    ]
+
+    # Try to cast a column that doesn't exist
+    csv_reader = CSVReader(
+        options=CSVReaderOptions(column_data_types={"NonExistent": "int64"})
+    )
+
+    # Should not raise an error for missing columns
+    df = csv_reader.file_to_dataframe(create_csv_file(csv_data))
+    assert df.shape == (2, 2)
+    assert df.columns.tolist() == ["Name", "City"]
+
+
+def test_csv_reader_error_detection_mixed_valid_invalid():
+    csv_data = [
+        ["Name", "Score", "City"],
+        ["name1", "95.5", "city1"],
+        ["name2", "87.2", "city2"],
+        ["name3", "92.1", "city3"],
+        ["name4", "eighty-five", "city4"],
+        ["name5", "78.9", "city5"],
+    ]
+
+    csv_reader = CSVReader(
+        options=CSVReaderOptions(column_data_types={"Score": "float64"})
+    )
+
+    with pytest.raises(DatabaseUploadFailed) as ex:
+        csv_reader.file_to_dataframe(create_csv_file(csv_data))
+
+    error_msg = str(ex.value)
+    assert "Cannot convert column 'Score' to float64" in error_msg
+    assert "Found 1 error(s):" in error_msg
+    assert "Line 5: 'eighty-five' cannot be converted to float64" in error_msg
+
+
+def test_csv_reader_error_detection_multiple_invalid_values():
+    """Test error detection with multiple invalid values showing first 5 + count."""
+    csv_data = [
+        ["Name", "Score", "City"],
+        ["name1", "95.5", "city1"],
+        ["name2", "87.2", "city2"],
+        ["name3", "92.1", "city3"],
+        ["name4", "eighty-five", "city4"],
+        ["name4", "eighty-one", "city4"],
+        ["name4", "eighty", "city4"],
+        ["name4", "one", "city4"],
+        ["name4", "two", "city4"],
+        ["name4", "three", "city4"],
+        ["name5", "78.9", "city5"],
+    ]
+
+    csv_reader = CSVReader(
+        options=CSVReaderOptions(column_data_types={"Score": "float64"})
+    )
+
+    with pytest.raises(DatabaseUploadFailed) as ex:
+        csv_reader.file_to_dataframe(create_csv_file(csv_data))
+
+    error_msg = str(ex.value)
+    assert "Cannot convert column 'Score' to float64" in error_msg
+    assert "Found 6 error(s):" in error_msg
+    assert "Line 5: 'eighty-five' cannot be converted to float64" in error_msg
+    assert "Line 6: 'eighty-one' cannot be converted to float64" in error_msg
+    assert "Line 7: 'eighty' cannot be converted to float64" in error_msg
+    assert "Line 8: 'one' cannot be converted to float64" in error_msg
+    assert "Line 9: 'two' cannot be converted to float64" in error_msg
+    assert "and 1 more error(s)" in error_msg
+
+
+def test_csv_reader_error_detection_non_numeric_types():
+    """Test error detection for non-numeric type casting."""
+    csv_data = [
+        ["Name", "Status", "City"],
+        ["name1", "active", "city1"],
+        ["name2", "inactive", "city2"],
+        ["name3", 123, "city3"],  # This should cause an error when casting to string
+    ]
+
+    csv_reader = CSVReader(
+        options=CSVReaderOptions(column_data_types={"Status": "string"})
+    )
+
+    # For non-numeric types, the error detection should still work
+    # but might have different behavior depending on pandas version
+    try:
+        df = csv_reader.file_to_dataframe(create_csv_file(csv_data))
+        # If no error is raised, the conversion succeeded
+        assert df["Status"].dtype == "string"
+    except DatabaseUploadFailed as ex:
+        # If an error is raised, it should have proper formatting
+        error_msg = str(ex.value)
+        assert "Cannot convert" in error_msg
+        assert "Status" in error_msg
+
+
+def test_csv_reader_error_detection_with_null_values():
+    csv_data = [
+        ["Name", "Age", "City"],
+        ["name1", "25", "city1"],
+        ["name2", "", "city2"],
+        ["name3", "invalid_age", "city3"],
+    ]
+
+    csv_reader = CSVReader(options=CSVReaderOptions(column_data_types={"Age": "int64"}))
+
+    with pytest.raises(DatabaseUploadFailed) as ex:
+        csv_reader.file_to_dataframe(create_csv_file(csv_data))
+
+    error_msg = str(ex.value)
+    assert "Cannot convert column 'Age' to int64" in error_msg
+    assert "Found 1 error(s):" in error_msg
+    assert "Line 4: 'invalid_age' cannot be converted to int64" in error_msg
+
+
+def test_csv_reader_successful_numeric_conversion():
+    csv_data = [
+        ["Name", "Age", "Score", "ID"],
+        ["name1", "25", "95.5", "1001"],
+        ["name2", "30", "87.2", "1002"],
+        ["name3", "35", "92.1", "1003"],
+    ]
+
+    csv_reader = CSVReader(
+        options=CSVReaderOptions(
+            column_data_types={
+                "Age": "int64",
+                "Score": "float64",
+                "ID": "int32",
+            }
+        )
+    )
+
+    df = csv_reader.file_to_dataframe(create_csv_file(csv_data))
+
+    assert df.shape == (3, 4)
+    assert df["Age"].dtype == "int64"
+    assert df["Score"].dtype == "float64"
+    assert df["ID"].dtype == "int32"
+    assert df.iloc[0]["Age"] == 25
+    assert df.iloc[0]["Score"] == 95.5
+    assert df.iloc[0]["ID"] == 1001
+
+
+def test_csv_reader_error_detection_improvements_summary():
+    csv_data_with_custom_header = [
+        ["metadata_row", "skip", "this"],
+        ["Name", "Age", "Score"],
+        ["Alice", "25", "95.5"],
+        ["Bob", "invalid_age", "87.2"],
+        ["Charlie", "30", "92.1"],
+    ]
+
+    csv_reader = CSVReader(
+        options=CSVReaderOptions(
+            header_row=1, column_data_types={"Age": "int64", "Score": "float64"}
+        )
+    )
+
+    with pytest.raises(DatabaseUploadFailed) as ex:
+        csv_reader.file_to_dataframe(create_csv_file(csv_data_with_custom_header))
+
+    error_msg = str(ex.value)
+    assert "Cannot convert column 'Age' to int64" in error_msg
+    assert "Found 1 error(s):" in error_msg
+    assert "Line 4: 'invalid_age' cannot be converted to int64" in error_msg
+
+    # Test case 2: Multiple type errors - Age comes first alphabetically
+    csv_data_multiple_errors = [
+        ["Name", "Age", "Score"],
+        ["Alice", "25", "95.5"],
+        ["Bob", "invalid_age", "invalid_score"],  # Error in both columns (line 3)
+        ["Charlie", "30", "92.1"],
+    ]
+
+    csv_reader = CSVReader(
+        options=CSVReaderOptions(column_data_types={"Age": "int64", "Score": "float64"})
+    )
+
+    with pytest.raises(DatabaseUploadFailed) as ex:
+        csv_reader.file_to_dataframe(create_csv_file(csv_data_multiple_errors))
+
+    error_msg = str(ex.value)
+    # Should catch the Age error first (Age comes before Score alphabetically)
+    assert "Cannot convert column 'Age' to int64" in error_msg
+    assert "Found 1 error(s):" in error_msg
+    assert "Line 3: 'invalid_age' cannot be converted to int64" in error_msg
+
+
+def test_csv_reader_cast_column_types_function():
+    """Test the _cast_column_types function directly for better isolation."""
+    # Create test DataFrame
+    test_data = {
+        "name": ["Alice", "Bob", "Charlie"],
+        "age": ["25", "30", "invalid_age"],
+        "score": ["95.5", "87.2", "92.1"],
+    }
+    df = pd.DataFrame(test_data)
+
+    # Test successful casting
+    types_success = {"age": "int64", "score": "float64"}
+    kwargs = {"header": 0}
+
+    # This should work for first two rows, but we'll only test the first two
+    df_subset = df.iloc[:2].copy()
+    result_df = CSVReader._cast_column_types(df_subset, types_success, kwargs)
+
+    assert result_df["age"].dtype == "int64"
+    assert result_df["score"].dtype == "float64"
+    assert result_df.iloc[0]["age"] == 25
+    assert result_df.iloc[0]["score"] == 95.5
+
+    # Test error case
+    with pytest.raises(DatabaseUploadFailed) as ex:
+        CSVReader._cast_column_types(df, types_success, kwargs)
+
+    error_msg = str(ex.value)
+    assert "Cannot convert column 'age' to int64" in error_msg
+    assert "Found 1 error(s):" in error_msg
+    assert "Line 4: 'invalid_age' cannot be converted to int64" in error_msg
+
+
+def test_csv_reader_cast_column_types_missing_column():
+    """Test _cast_column_types with missing columns."""
+    test_data = {
+        "name": ["Alice", "Bob"],
+        "age": ["25", "30"],
+    }
+    df = pd.DataFrame(test_data)
+
+    # Try to cast a column that doesn't exist
+    types = {"age": "int64", "nonexistent": "float64"}
+    kwargs = {"header": 0}
+
+    # Should not raise an error for missing columns
+    result_df = CSVReader._cast_column_types(df, types, kwargs)
+    assert result_df["age"].dtype == "int64"
+    assert "nonexistent" not in result_df.columns
+
+
+def test_csv_reader_cast_column_types_different_numeric_types():
+    """Test _cast_column_types with various numeric types."""
+    test_data = {
+        "int32_col": ["1", "2", "3"],
+        "int64_col": ["100", "200", "300"],
+        "float32_col": ["1.5", "2.5", "3.5"],
+        "float64_col": ["10.1", "20.2", "30.3"],
+    }
+    df = pd.DataFrame(test_data)
+
+    types = {
+        "int32_col": "int32",
+        "int64_col": "int64",
+        "float32_col": "float32",
+        "float64_col": "float64",
+    }
+    kwargs = {"header": 0}
+
+    result_df = CSVReader._cast_column_types(df, types, kwargs)
+
+    assert result_df["int32_col"].dtype == "int32"
+    assert result_df["int64_col"].dtype == "int64"
+    assert result_df["float32_col"].dtype == "float32"
+    assert result_df["float64_col"].dtype == "float64"
+
+
 def test_csv_reader_chunking_large_file():
     """Test that chunking is used for large files."""
     # Create a large CSV with more than 100k rows