- dr.across() with dr.mutate()
- dr.across(f.col1 | f.col2): apply functions to specific columns
- dr.across(dr.where() | &): apply functions to columns that meet certain conditions
- dr.across(dr.everything()): apply a function to all columns
==================================
- dr.across() with dr.summarise()
- dr.across(dr.c(f.col1, f.col2))): apply functions to specific columns
- dr.across(dr.where() & |): apply functions to columns that meet certain conditions
- dr.across(dr.everything()): apply a function to all columns
==================================
- dr.across() with dr.reframe()
dr.across(
cols,
func=lambda col: ...,
_names="{_col}_new" # _col is a placeholder for the original
)
In [2]:
Copied!
import datar.all as dr
from datar import f
import pandas as pd
import numpy as np
from pathlib import Path
pd.set_option("display.width", 200)
# Get the path object pointing to the ``notebooks`` directory that contains *.csv files
data_dir = next(Path("/home").rglob("*/notebooks/*.csv")).parent
import datar.all as dr
from datar import f
import pandas as pd
import numpy as np
from pathlib import Path
pd.set_option("display.width", 200)
# Get the path object pointing to the ``notebooks`` directory that contains *.csv files
data_dir = next(Path("/home").rglob("*/notebooks/*.csv")).parent
In [11]:
Copied!
tb_pokemon = dr.tibble(
pd.read_csv(data_dir/"pokemon.csv")
>> dr.rename_with(lambda col: col.strip().replace(" ", "_").replace(".", "")) # Clean column names
>> dr.select(~f["#"]) # Drop the "#" column
>> dr.mutate(
Type_1 = f.Type_1.astype("category"), # convert to category (pandas style)
Type_2 = dr.as_factor(f.Type_2), # convert to category (datar style)
Generation = dr.as_ordered(f.Generation), # convert to ordered category (datar style)
Legendary = dr.as_logical(f.Legendary) # convert to boolean (datar style)
)
)
print(
tb_pokemon
>> dr.slice_head(n=5)
)
tb_pokemon = dr.tibble(
pd.read_csv(data_dir/"pokemon.csv")
>> dr.rename_with(lambda col: col.strip().replace(" ", "_").replace(".", "")) # Clean column names
>> dr.select(~f["#"]) # Drop the "#" column
>> dr.mutate(
Type_1 = f.Type_1.astype("category"), # convert to category (pandas style)
Type_2 = dr.as_factor(f.Type_2), # convert to category (datar style)
Generation = dr.as_ordered(f.Generation), # convert to ordered category (datar style)
Legendary = dr.as_logical(f.Legendary) # convert to boolean (datar style)
)
)
print(
tb_pokemon
>> dr.slice_head(n=5)
)
Name Type_1 Type_2 Total HP Attack Defense Sp_Atk Sp_Def Speed Generation Legendary
<str> <category> <category> <int64> <int64> <int64> <int64> <int64> <int64> <int64> <category> <bool>
0 Bulbasaur Grass Poison 318 45 49 49 65 65 45 1 False
1 Ivysaur Grass Poison 405 60 62 63 80 80 60 1 False
2 Venusaur Grass Poison 525 80 82 83 100 100 80 1 False
3 VenusaurMega Venusaur Grass Poison 625 80 100 123 122 120 80 1 False
4 Charmander Fire NaN 309 39 52 43 60 50 65 1 False
1. dr.across() with dr.mutate()¶
In [12]:
Copied!
##-------------------------------------------------------##
## dr.across(f.col1 | f.col2) to modify specific columns ##
##-------------------------------------------------------##
#--------------------------------
## Use dr.across(f.col1 | f.col2) to lowercase
#--------------------------------
print(
tb_pokemon
>> dr.mutate(
dr.across(
f.Name | f.Type_1 | f.Type_2, # Apply to specific columns
dr.tolower # Function to apply (lowercase)
)
)
>> dr.slice_head(n=5)
>> dr.select(f.Name, f.Type_1, f.Type_2) # Only show modified columns
)
##-------------------------------------------------------##
## dr.across(f.col1 | f.col2) to modify specific columns ##
##-------------------------------------------------------##
#--------------------------------
## Use dr.across(f.col1 | f.col2) to lowercase
#--------------------------------
print(
tb_pokemon
>> dr.mutate(
dr.across(
f.Name | f.Type_1 | f.Type_2, # Apply to specific columns
dr.tolower # Function to apply (lowercase)
)
)
>> dr.slice_head(n=5)
>> dr.select(f.Name, f.Type_1, f.Type_2) # Only show modified columns
)
Name Type_1 Type_2
<str> <str> <str>
0 bulbasaur grass poison
1 ivysaur grass poison
2 venusaur grass poison
3 venusaurmega venusaur grass poison
4 charmander fire nan
In [6]:
Copied!
#--------------------------------
## Use dr.across(f.col1 | f.col2) to convert to float
#--------------------------------
print(
tb_pokemon
>> dr.mutate(
dr.across(
f.HP | f.Attack | f.Defense, # Apply to specific
lambda col: col.astype(float) # Function to apply (convert to float)
)
)
>> dr.slice_head(n=5)
>> dr.select(f.HP, f.Attack, f.Defense) # Only show modified columns
)
#--------------------------------
## Use dr.across(f.col1 | f.col2) to convert to float
#--------------------------------
print(
tb_pokemon
>> dr.mutate(
dr.across(
f.HP | f.Attack | f.Defense, # Apply to specific
lambda col: col.astype(float) # Function to apply (convert to float)
)
)
>> dr.slice_head(n=5)
>> dr.select(f.HP, f.Attack, f.Defense) # Only show modified columns
)
HP Attack Defense <float64> <float64> <float64> 0 45.0 49.0 49.0 1 60.0 62.0 63.0 2 80.0 82.0 83.0 3 80.0 100.0 123.0 4 39.0 52.0 43.0
In [7]:
Copied!
#--------------------------------
## Use dr.across(f.col1 | f.col2) to convert to ordered categorical
#--------------------------------
print(
tb_pokemon
>> dr.mutate(
dr.across(
f.Generation | f.Legendary, # Apply to specific columns
dr.as_ordered # Convert to ordered categorical
)
)
>> dr.slice_head(n=5)
>> dr.select(f.Name, f.Generation, f.Legendary)
)
#--------------------------------
## Use dr.across(f.col1 | f.col2) to convert to ordered categorical
#--------------------------------
print(
tb_pokemon
>> dr.mutate(
dr.across(
f.Generation | f.Legendary, # Apply to specific columns
dr.as_ordered # Convert to ordered categorical
)
)
>> dr.slice_head(n=5)
>> dr.select(f.Name, f.Generation, f.Legendary)
)
Name Generation Legendary
<str> <category> <category>
0 Bulbasaur 1 False
1 Ivysaur 1 False
2 Venusaur 1 False
3 VenusaurMega Venusaur 1 False
4 Charmander 1 False
In [21]:
Copied!
##-------------------------------------------------------------------##
## Use dr.across(dr.where() | &) to modify cols satisfied conditions ##
##-------------------------------------------------------------------##
def min_max_scaler(series):
series = pd.Series(series) # Ensure input is a pandas Series
return (series - series.min()) / (series.max() - series.min())
#--------------------------------
## Use dr.across(dr.where(dr.is_numeric) & ~f["Generation", "Legendary"])
#--------------------------------
print(
tb_pokemon
>> dr.mutate(
dr.across(
dr.where(dr.is_numeric) & (~f["Generation", "Legendary"]), # Apply to all numeric columns (except "Generation" and "Legendary")
min_max_scaler # Function to apply
)
)
>> dr.slice_head(n=5)
)
##-------------------------------------------------------------------##
## Use dr.across(dr.where() | &) to modify cols satisfied conditions ##
##-------------------------------------------------------------------##
def min_max_scaler(series):
series = pd.Series(series) # Ensure input is a pandas Series
return (series - series.min()) / (series.max() - series.min())
#--------------------------------
## Use dr.across(dr.where(dr.is_numeric) & ~f["Generation", "Legendary"])
#--------------------------------
print(
tb_pokemon
>> dr.mutate(
dr.across(
dr.where(dr.is_numeric) & (~f["Generation", "Legendary"]), # Apply to all numeric columns (except "Generation" and "Legendary")
min_max_scaler # Function to apply
)
)
>> dr.slice_head(n=5)
)
Name Type_1 Type_2 Total HP Attack Defense Sp_Atk Sp_Def Speed Generation Legendary
<str> <category> <category> <float64> <float64> <float64> <float64> <float64> <float64> <float64> <category> <bool>
0 Bulbasaur Grass Poison 0.230000 0.173228 0.237838 0.195556 0.298913 0.214286 0.228571 1 False
1 Ivysaur Grass Poison 0.375000 0.232283 0.308108 0.257778 0.380435 0.285714 0.314286 1 False
2 Venusaur Grass Poison 0.575000 0.311024 0.416216 0.346667 0.489130 0.380952 0.428571 1 False
3 VenusaurMega Venusaur Grass Poison 0.741667 0.311024 0.513514 0.524444 0.608696 0.476190 0.428571 1 False
4 Charmander Fire NaN 0.215000 0.149606 0.254054 0.168889 0.271739 0.142857 0.342857 1 False
In [22]:
Copied!
#--------------------------------
## Use dr.across(dr.where(dr.is_character) & ~f.Name)
#--------------------------------
print(
tb_pkm_chr
>> dr.mutate(
dr.across(
dr.where(dr.is_character) & (~f.Name), # Apply to all string columns except the "Name"
lambda col: col.str.lower() # Function to apply
)
)
>> dr.slice_head(n=5)
)
print('''
NOTICE: ``lambda col: col.str.lower()`` uses Pandas API.
In this case, since "Type_2" has NaN values, these NaN stay the same,
they are not converted to "nan".
(Because Pandas API does not handle that)
''')
#--------------------------------
## Use dr.across(dr.where(dr.is_character) & ~f.Name)
#--------------------------------
print(
tb_pkm_chr
>> dr.mutate(
dr.across(
dr.where(dr.is_character) & (~f.Name), # Apply to all string columns except the "Name"
lambda col: col.str.lower() # Function to apply
)
)
>> dr.slice_head(n=5)
)
print('''
NOTICE: ``lambda col: col.str.lower()`` uses Pandas API.
In this case, since "Type_2" has NaN values, these NaN stay the same,
they are not converted to "nan".
(Because Pandas API does not handle that)
''')
Name Type_1 Type_2 Total HP Attack Defense Sp_Atk Sp_Def Speed Generation Legendary
<str> <str> <str> <int64> <int64> <int64> <int64> <int64> <int64> <int64> <category> <bool>
0 Bulbasaur grass poison 318 45 49 49 65 65 45 1 False
1 Ivysaur grass poison 405 60 62 63 80 80 60 1 False
2 Venusaur grass poison 525 80 82 83 100 100 80 1 False
3 VenusaurMega Venusaur grass poison 625 80 100 123 122 120 80 1 False
4 Charmander fire NaN 309 39 52 43 60 50 65 1 False
NOTICE: ``lambda col: col.str.lower()`` uses Pandas API.
In this case, since "Type_2" has NaN values, these NaN stay the same,
they are not converted to "nan".
(Because Pandas API does not handle that)
In [23]:
Copied!
print(
tb_pokemon
>> dr.mutate(
dr.across(
dr.where(dr.is_character) & (~f.Name), # Apply to all string columns except the "Name"
dr.tolower # Function to apply
)
)
>> dr.slice_head(n=5)
)
print('''
NOTICE: ``dr.tolower`` uses DataR API.
In this case, NaN values in ``Type_2`` are now converted to "nan"
(Because DataR API can handle that)
''')
print(
tb_pokemon
>> dr.mutate(
dr.across(
dr.where(dr.is_character) & (~f.Name), # Apply to all string columns except the "Name"
dr.tolower # Function to apply
)
)
>> dr.slice_head(n=5)
)
print('''
NOTICE: ``dr.tolower`` uses DataR API.
In this case, NaN values in ``Type_2`` are now converted to "nan"
(Because DataR API can handle that)
''')
Name Type_1 Type_2 Total HP Attack Defense Sp_Atk Sp_Def Speed Generation Legendary
<str> <str> <str> <int64> <int64> <int64> <int64> <int64> <int64> <int64> <category> <bool>
0 Bulbasaur grass poison 318 45 49 49 65 65 45 1 False
1 Ivysaur grass poison 405 60 62 63 80 80 60 1 False
2 Venusaur grass poison 525 80 82 83 100 100 80 1 False
3 VenusaurMega Venusaur grass poison 625 80 100 123 122 120 80 1 False
4 Charmander fire nan 309 39 52 43 60 50 65 1 False
NOTICE: ``dr.tolower`` uses DataR API.
In this case, NaN values in ``Type_2`` are now converted to "nan"
(Because DataR API can handle that)
In [24]:
Copied!
##--------------------------------------------------##
## dr.across(dr.everything()) to modify all columns ##
##--------------------------------------------------##
print(
tb_pokemon
>> dr.mutate(
dr.across(
dr.everything(), # Apply to all columns
dr.as_character # Convert all cols to string
)
)
>> dr.slice_head(n=5)
)
##--------------------------------------------------##
## dr.across(dr.everything()) to modify all columns ##
##--------------------------------------------------##
print(
tb_pokemon
>> dr.mutate(
dr.across(
dr.everything(), # Apply to all columns
dr.as_character # Convert all cols to string
)
)
>> dr.slice_head(n=5)
)
Name Type_1 Type_2 Total HP Attack Defense Sp_Atk Sp_Def Speed Generation Legendary
<str> <str> <str> <str> <str> <str> <str> <str> <str> <str> <str> <str>
0 Bulbasaur Grass Poison 318 45 49 49 65 65 45 1 False
1 Ivysaur Grass Poison 405 60 62 63 80 80 60 1 False
2 Venusaur Grass Poison 525 80 82 83 100 100 80 1 False
3 VenusaurMega Venusaur Grass Poison 625 80 100 123 122 120 80 1 False
4 Charmander Fire NaN 309 39 52 43 60 50 65 1 False
2. dr.across() with dr.summarise()¶
In [25]:
Copied!
##-------------------------------------------##
## dr.across(f.col1 | f.col2): specific cols ##
##-------------------------------------------##
print(
tb_pokemon
>> dr.summarise(
dr.across(
f.HP | f.Attack | f.Defense | f.Sp_Atk | f.Sp_Def | f.Speed,
dr.mean,
_names="avg_{_col}" # _col is a placeholder for the original column name
)
)
)
##-------------------------------------------##
## dr.across(f.col1 | f.col2): specific cols ##
##-------------------------------------------##
print(
tb_pokemon
>> dr.summarise(
dr.across(
f.HP | f.Attack | f.Defense | f.Sp_Atk | f.Sp_Def | f.Speed,
dr.mean,
_names="avg_{_col}" # _col is a placeholder for the original column name
)
)
)
avg_HP avg_Attack avg_Defense avg_Sp_Atk avg_Sp_Def avg_Speed <float64> <float64> <float64> <float64> <float64> <float64> 0 69.25875 79.00125 73.8425 72.82 71.9025 68.2775
In [26]:
Copied!
##---------------------------------------------##
## dr.across(dr.where() & |): conditional cols ##
##---------------------------------------------##
print(
tb_pokemon
>> dr.summarise(
dr.across(
dr.where(dr.is_numeric) & (~f.Legendary), # All numeric columns except 'Legendary'
lambda col: np.quantile(col, [0.25, 0.5, 0.75]),
_names="quantile_{_col}" # _col is a placeholder for the original column name
)
)
>> dr.mutate(id = ["Q1", "Median", "Q3"]) # Add an id column for the quantiles
>> dr.column_to_rownames("id") # Set the id column as row names
)
##---------------------------------------------##
## dr.across(dr.where() & |): conditional cols ##
##---------------------------------------------##
print(
tb_pokemon
>> dr.summarise(
dr.across(
dr.where(dr.is_numeric) & (~f.Legendary), # All numeric columns except 'Legendary'
lambda col: np.quantile(col, [0.25, 0.5, 0.75]),
_names="quantile_{_col}" # _col is a placeholder for the original column name
)
)
>> dr.mutate(id = ["Q1", "Median", "Q3"]) # Add an id column for the quantiles
>> dr.column_to_rownames("id") # Set the id column as row names
)
quantile_Total quantile_HP quantile_Attack quantile_Defense quantile_Sp_Atk quantile_Sp_Def quantile_Speed
<float64> <float64> <float64> <float64> <float64> <float64> <float64>
Q1 330.0 50.0 55.0 50.0 49.75 50.0 45.0
Median 450.0 65.0 75.0 70.0 65.00 70.0 65.0
Q3 515.0 80.0 100.0 90.0 95.00 90.0 90.0
In [27]:
Copied!
##-------------------------------------------------------------##
## dr.across(dr.everything()): apply a function to all columns ##
##-------------------------------------------------------------##
#------------------------
## Count NA values in each column
#------------------------
print(
tb_pokemon
>> dr.summarise(
dr.across(
dr.everything(), # Apply to all columns
lambda col: col.isna().sum() # Function to apply (count NA values in each column)
)
)
)
##-------------------------------------------------------------##
## dr.across(dr.everything()): apply a function to all columns ##
##-------------------------------------------------------------##
#------------------------
## Count NA values in each column
#------------------------
print(
tb_pokemon
>> dr.summarise(
dr.across(
dr.everything(), # Apply to all columns
lambda col: col.isna().sum() # Function to apply (count NA values in each column)
)
)
)
Name Type_1 Type_2 Total HP Attack Defense Sp_Atk Sp_Def Speed Generation Legendary <int64> <int64> <int64> <int64> <int64> <int64> <int64> <int64> <int64> <int64> <int64> <int64> 0 0 0 386 0 0 0 0 0 0 0 0 0
In [28]:
Copied!
#------------------------
## Count unique values in each column
#------------------------
print(
tb_pokemon
>> dr.summarise(
dr.across(
dr.everything(), # Apply to all columns
lambda col: len(col.unique()) # Function to apply (count unique values in each column)
)
)
)
#------------------------
## Count unique values in each column
#------------------------
print(
tb_pokemon
>> dr.summarise(
dr.across(
dr.everything(), # Apply to all columns
lambda col: len(col.unique()) # Function to apply (count unique values in each column)
)
)
)
Name Type_1 Type_2 Total HP Attack Defense Sp_Atk Sp_Def Speed Generation Legendary <int64> <int64> <int64> <int64> <int64> <int64> <int64> <int64> <int64> <int64> <int64> <int64> 0 800 18 19 200 94 111 103 105 92 108 6 2
3. dr.across() with dr.reframe()¶
'''Apply the same reframing function to multiple columns of the DataFrame.'''
In [29]:
Copied!
from scipy import stats
tb_pokemon = dr.tibble(
pd.read_csv(
filepath_or_buffer=data_dir/"pokemon.csv",
dtype={
"Type 1": "category",
"Type 2": "category",
"Generation": "category"
}
)
.pipe(lambda f: f.set_axis(f.columns.str.strip().str.replace(r"\s+", "_", regex=True).str.replace(".", ""), axis=1))
.drop(columns=["Total", "#", "Sp_Atk", "Sp_Def", "Legendary"])
.assign(Generation = lambda f: f['Generation'].cat.as_ordered())
)
from scipy import stats
tb_pokemon = dr.tibble(
pd.read_csv(
filepath_or_buffer=data_dir/"pokemon.csv",
dtype={
"Type 1": "category",
"Type 2": "category",
"Generation": "category"
}
)
.pipe(lambda f: f.set_axis(f.columns.str.strip().str.replace(r"\s+", "_", regex=True).str.replace(".", ""), axis=1))
.drop(columns=["Total", "#", "Sp_Atk", "Sp_Def", "Legendary"])
.assign(Generation = lambda f: f['Generation'].cat.as_ordered())
)
In [30]:
Copied!
##-------------------------------------------------------------##
## Example 1: calculate the Shapiro-Wilk test for some columns ##
##-------------------------------------------------------------##
print(
tb_pokemon
>> dr.reframe(
dr.across(
f.Defense | f.Speed | f.Attack, # specify multiple columns with | (bitwise or)
lambda col: stats.shapiro(col),
_names="{_col}_normality" # _col is a placeholder for the original column name
)
)
>> dr.pipe(lambda f: f.set_axis(["W-statistic", "p-value"], axis=0)) # rename the index
)
##-------------------------------------------------------------##
## Example 1: calculate the Shapiro-Wilk test for some columns ##
##-------------------------------------------------------------##
print(
tb_pokemon
>> dr.reframe(
dr.across(
f.Defense | f.Speed | f.Attack, # specify multiple columns with | (bitwise or)
lambda col: stats.shapiro(col),
_names="{_col}_normality" # _col is a placeholder for the original column name
)
)
>> dr.pipe(lambda f: f.set_axis(["W-statistic", "p-value"], axis=0)) # rename the index
)
Defense_normality Speed_normality Attack_normality
<float64> <float64> <float64>
W-statistic 9.380628e-01 9.841602e-01 9.789301e-01
p-value 9.923172e-18 1.309542e-07 2.472154e-09
In [31]:
Copied!
##-------------------------------------------------##
## Example 2: calculate quantiles for some columns ##
##-------------------------------------------------##
print(
tb_pokemon
>> dr.reframe(
dr.across(
dr.where(dr.is_numeric),
lambda col: np.quantile(col, q=[0.25, 0.5, 0.75, 1]),
_names="{_col}_quantiles"
)
)
>> dr.pipe(lambda f: f.set_axis(["Q1", "Q2", "Q3", "Q4"], axis=0)) # rename the index
)
##-------------------------------------------------##
## Example 2: calculate quantiles for some columns ##
##-------------------------------------------------##
print(
tb_pokemon
>> dr.reframe(
dr.across(
dr.where(dr.is_numeric),
lambda col: np.quantile(col, q=[0.25, 0.5, 0.75, 1]),
_names="{_col}_quantiles"
)
)
>> dr.pipe(lambda f: f.set_axis(["Q1", "Q2", "Q3", "Q4"], axis=0)) # rename the index
)
HP_quantiles Attack_quantiles Defense_quantiles Speed_quantiles
<float64> <float64> <float64> <float64>
Q1 50.0 55.0 50.0 45.0
Q2 65.0 75.0 70.0 65.0
Q3 80.0 100.0 90.0 90.0
Q4 255.0 190.0 230.0 180.0