pandas の基本操作

from pathlib import Path
import pandas as pd

DATA = Path("../data/processed/soumu-gdp-clean.csv")
df = pd.read_csv(DATA)
df.head()

df.info()
df.describe().T[['mean','std','min','max']].round(2).head()

df.columns = (
    df.columns
      .str.replace("年度", "")
      .str.replace("令和", "R")
      .str.replace("平", "H")
)

df.loc[df["カテゴリ"] == "社会保障関係費", ["カテゴリ", "H1", "H10", "R1"]]
df.iloc[:3, :5]  # 位置ベース

def pick(category, cols):
    return df.loc[df["カテゴリ"] == category, cols]

pick("教育費", ["H1", "H5", "R1"])

mask = (df["R1"] - df["H1"]) > 5
growth = (
    df.assign(
        diff = df["R1"] - df["H1"],
        avg  = df.filter(regex="^(H|R)").mean(axis=1)
    )
    .loc[mask, ["カテゴリ", "diff", "avg"]]
    .sort_values("diff", ascending=False)
)
growth.head()

value_cols = df.columns[df.columns != "カテゴリ"]

tidy = (
    df.melt(
        id_vars="カテゴリ",
        value_vars=value_cols,
        var_name="year",
        value_name="value"
    )
    .assign(
        year = lambda x: x["year"].str.replace("H", "19").str.replace("R", "20").astype(int)
    )
)
tidy.head()

pivot = tidy.pivot_table(
    index="year",
    columns="カテゴリ",
    values="value"
)

df.to_parquet("../data/processed/soumu-gdp-clean.parquet", index=False)
tidy.to_csv("../data/processed/soumu-gdp-tidy.csv", index=False, encoding="utf-8-sig")