前言
在数据科学领域，Pandas 是 Python 生态中最核心的工具之一。无论是数据清洗、转换还是可视化准备，Pandas 都提供了高效且直观的 API。本文将通过实际案例，系统地讲解 Pandas 的常用操作和进阶技巧。
环境搭建
安装
1
2
3
4
5
6
|
pip install pandas
pip install pandas numpy matplotlib
|
导入和数据创建
1
2
3
4
5
6
7
8
9
10
11
12
13
| import pandas as pd
import numpy as np
df = pd.DataFrame({
'name': ['Alice', 'Bob', 'Charlie', 'Diana', 'Eve'],
'age': [25, 30, 35, 28, 32],
'city': ['Beijing', 'Shanghai', 'Guangzhou', 'Shenzhen', 'Beijing'],
'salary': [15000, 22000, 18000, 25000, 19000],
'department': ['Engineering', 'Marketing', 'Engineering', 'Sales', 'Marketing']
})
print(df.head())
|
Series 与 DataFrame
| 数据结构 |
维度 |
类比 |
| Series |
1 维 |
Excel 中的一列 |
| DataFrame |
2 维 |
Excel 中的一张表 |
| Panel |
3 维 |
已弃用 |
数据读取与写入
文件格式支持
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
|
df = pd.read_csv('data.csv', encoding='utf-8')
df = pd.read_excel('data.xlsx', sheet_name='Sheet1')
df = pd.read_json('data.json')
from sqlalchemy import create_engine
engine = create_engine('sqlite:///database.db')
df = pd.read_sql('SELECT * FROM users', engine)
df.to_csv('output.csv', index=False)
df.to_excel('output.xlsx', sheet_name='Results')
df.to_json('output.json', orient='records')
|
读取大数据集的技巧
1
2
3
4
5
6
7
8
9
10
11
12
|
chunk_size = 10000
chunks = []
for chunk in pd.read_csv('large_file.csv', chunksize=chunk_size):
filtered = chunk[chunk['value'] > 0]
chunks.append(filtered)
df_large = pd.concat(chunks, ignore_index=True)
df = pd.read_csv('data.csv', usecols=['name', 'salary', 'city'])
|
数据探索
基本信息查看
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
|
df.head(10)
df.info()
df.describe()
df.shape
df.columns.tolist()
df.dtypes
df['city'].unique()
df['city'].value_counts()
|
缺失值检测
1
2
3
4
5
6
7
8
9
|
df.isnull().sum()
(df.isnull().sum() / len(df)) * 100
import missingno as msno
msno.matrix(df)
|
数据选择与过滤
基本选择
1
2
3
4
5
6
7
8
9
10
11
12
13
|
df['name']
df[['name', 'age']]
df.loc[0]
df.loc[0:5, 'name':'age']
df.loc[df['age'] > 30]
df.iloc[0]
df.iloc[:5, :3]
df.iloc[0, 0]
|
高级过滤
1
2
3
4
5
6
7
8
9
10
11
12
13
14
|
filtered = df[
(df['age'] >= 25) &
(df['age'] <= 35) &
(df['city'].isin(['Beijing', 'Shanghai']))
]
df[df['name'].str.contains('A')]
df[df['name'].str.startswith('B')]
df[df['name'].str.len() > 4]
result = df.query('age > 25 and salary >= 20000 and city == "Shanghai"')
|
数据清洗
处理缺失值
1
2
3
4
5
6
7
8
9
10
11
|
df_cleaned = df.dropna()
df_cleaned = df.dropna(axis=1)
df['age'].fillna(df['age'].mean(), inplace=True)
df['city'].fillna('Unknown', inplace=True)
df['salary'].fillna(method='ffill', inplace=True)
df['salary'].fillna(method='bfill', inplace=True)
|
处理重复值
1
2
3
4
5
6
7
8
|
df.duplicated().sum()
df.drop_duplicates(inplace=True)
df.drop_duplicates(subset=['name', 'email'], keep='first', inplace=True)
|
数据类型转换
1
2
3
4
5
6
7
|
df['age'] = df['age'].astype(int)
df['date'] = pd.to_datetime(df['date'])
df['price'] = pd.to_numeric(df['price'], errors='coerce')
df['city'] = df['city'].astype('category')
|
数据变换
apply 函数
1
2
3
4
5
6
7
8
9
10
11
12
13
|
df['salary_level'] = df['salary'].apply(
lambda x: 'High' if x >= 20000 else 'Low'
)
def calculate_bonus(row):
if row['department'] == 'Sales':
return row['salary'] * 0.2
else:
return row['salary'] * 0.1
df['bonus'] = df.apply(calculate_bonus, axis=1)
|
创建新列
1
2
3
4
5
6
7
8
9
10
11
12
13
|
df['age_group'] = pd.cut(
df['age'],
bins=[0, 25, 35, 45, 100],
labels=['Young', 'Mid', 'Senior', 'Veteran']
)
df['dept_code'] = df['department'].map({
'Engineering': 'ENG',
'Marketing': 'MKT',
'Sales': 'SLS'
})
|
数据聚合
GroupBy
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
|
dept_stats = df.groupby('department').agg({
'salary': ['mean', 'median', 'max', 'min', 'count'],
'age': 'mean'
})
def salary_range(x):
return x.max() - x.min()
city_stats = df.groupby('city').agg(
total_employees=('name', 'count'),
avg_salary=('salary', 'mean'),
salary_spread=('salary', salary_range)
).reset_index()
print(city_stats)
|
透视表
1
2
3
4
5
6
7
8
9
|
pivot = pd.pivot_table(
df,
values='salary',
index='department',
columns='age_group',
aggfunc='mean',
fill_value=0
)
|
窗口函数
1
2
3
4
5
6
7
8
9
10
|
df['salary_ma_3'] = df.groupby('department')['salary'].transform(
lambda x: x.rolling(window=3, min_periods=1).mean()
)
df['cumulative_salary'] = df.groupby('department')['salary'].cumsum()
df['rank'] = df.groupby('department')['salary'].rank(ascending=False)
|
数据合并
各种 Join
1
2
3
4
5
6
7
8
9
10
11
|
merged = pd.merge(df1, df2, on='user_id', how='inner')
merged = pd.merge(df1, df2, on='user_id', how='left')
merged = pd.merge(df1, df2, on='user_id', how='outer')
combined = pd.concat([df_2024, df_2025], ignore_index=True)
|
性能优化
数据类型优化
1
2
3
4
5
6
7
8
9
10
11
|
df.memory_usage(deep=True)
df['age'] = pd.to_numeric(df['age'], downcast='integer')
df['salary'] = pd.to_numeric(df['salary'], downcast='float')
df['city'] = df['city'].astype('category')
|
向量化操作
1
2
3
4
5
6
|
for i in range(len(df)):
df.loc[i, 'bonus'] = df.loc[i, 'salary'] * 0.1
df['bonus'] = df['salary'] * 0.1
|
实战案例：销售数据分析
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
|
import pandas as pd
import matplotlib.pyplot as plt
sales = pd.read_csv('sales_2024.csv', parse_dates=['date'])
sales.dropna(subset=['amount', 'product'], inplace=True)
sales['month'] = sales['date'].dt.month
monthly = sales.groupby('month').agg(
total_revenue=('amount', 'sum'),
order_count=('order_id', 'nunique'),
avg_order_value=('amount', 'mean')
).reset_index()
top_products = sales.groupby('product')['amount'].sum().nlargest(10)
print("=== 月度销售概况 ===")
print(monthly)
print(f"\n=== Top 10 产品 ===")
print(top_products)
|
总结
Pandas 是数据分析的瑞士军刀。掌握本文介绍的核心操作（数据读取、清洗、变换、聚合、合并），足以应对日常 90% 的数据分析需求。建议在实际项目中多加练习，尽快熟悉这些 API。
