書籍「scikit-learn、Keras、TensorFlowによる実践機械学習 第2版」写経＆自分用メモ

```python
# Python ≥3.5 is required
import sys
assert sys.version_info >= (3, 5)

# Scikit-Learn ≥0.20 is required
import sklearn
assert sklearn.__version__ >= "0.20"

# Common imports
import numpy as np
import os

# to make this notebook's output stable across runs
np.random.seed(42)

# To plot pretty figures
%matplotlib inline
import matplotlib as mpl
import matplotlib.pyplot as plt
mpl.rc('axes', labelsize=14)
mpl.rc('xtick', labelsize=12)
mpl.rc('ytick', labelsize=12)

# Where to save the figures
PROJECT_ROOT_DIR = "."
CHAPTER_ID = "end_to_end_project"
IMAGES_PATH = os.path.join(PROJECT_ROOT_DIR, "images", CHAPTER_ID)
os.makedirs(IMAGES_PATH, exist_ok=True)

def save_fig(fig_id, tight_layout=True, fig_extension="png", resolution=300):
    path = os.path.join(IMAGES_PATH, fig_id + "." + fig_extension)
    print("Saving figure", fig_id)
    if tight_layout:
        plt.tight_layout()
    plt.savefig(path, format=fig_extension, dpi=resolution)

# Ignore useless warnings (see SciPy issue #5998)
import warnings
warnings.filterwarnings(action="ignore", message="^internal gelsd")
```


```python
import tarfile
import urllib
DOWNLOAD_ROOT = "https://raw.githubusercontent.com/ageron/handson-ml2/master/"
HOUSING_PATH = os.path.join("datasets", "housing")
HOUSING_URL = DOWNLOAD_ROOT + "datasets/housing/housing.tgz"

def fetch_housing_data(housing_url=HOUSING_URL, housing_path=HOUSING_PATH):
    if not os.path.isdir(housing_path):
        os.makedirs(housing_path)
    tgz_path = os.path.join(housing_path, "housing.tgz")
    urllib.request.urlretrieve(housing_url, tgz_path)
    housing_tgz = tarfile.open(tgz_path)
    housing_tgz.extractall(path=housing_path)
    housing_tgz.close()
```


```python
fetch_housing_data()
```


```python
import pandas as pd

def load_housing_data(housing_path=HOUSING_PATH):
    csv_path = os.path.join(housing_path, "housing.csv")
    return pd.read_csv(csv_path)
```

# データ構造を見てみる


```python
housing = load_housing_data()
housing.head()
```




<div>
<style scoped>
    .dataframe tbody tr th:only-of-type {
        vertical-align: middle;
    }

    .dataframe tbody tr th {
        vertical-align: top;
    }

    .dataframe thead th {
        text-align: right;
    }
</style>
<table border="1" class="dataframe">
  <thead>
    <tr style="text-align: right;">
      <th></th>
      <th>longitude</th>
      <th>latitude</th>
      <th>housing_median_age</th>
      <th>total_rooms</th>
      <th>total_bedrooms</th>
      <th>population</th>
      <th>households</th>
      <th>median_income</th>
      <th>median_house_value</th>
      <th>ocean_proximity</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <th>0</th>
      <td>-122.23</td>
      <td>37.88</td>
      <td>41.0</td>
      <td>880.0</td>
      <td>129.0</td>
      <td>322.0</td>
      <td>126.0</td>
      <td>8.3252</td>
      <td>452600.0</td>
      <td>NEAR BAY</td>
    </tr>
    <tr>
      <th>1</th>
      <td>-122.22</td>
      <td>37.86</td>
      <td>21.0</td>
      <td>7099.0</td>
      <td>1106.0</td>
      <td>2401.0</td>
      <td>1138.0</td>
      <td>8.3014</td>
      <td>358500.0</td>
      <td>NEAR BAY</td>
    </tr>
    <tr>
      <th>2</th>
      <td>-122.24</td>
      <td>37.85</td>
      <td>52.0</td>
      <td>1467.0</td>
      <td>190.0</td>
      <td>496.0</td>
      <td>177.0</td>
      <td>7.2574</td>
      <td>352100.0</td>
      <td>NEAR BAY</td>
    </tr>
    <tr>
      <th>3</th>
      <td>-122.25</td>
      <td>37.85</td>
      <td>52.0</td>
      <td>1274.0</td>
      <td>235.0</td>
      <td>558.0</td>
      <td>219.0</td>
      <td>5.6431</td>
      <td>341300.0</td>
      <td>NEAR BAY</td>
    </tr>
    <tr>
      <th>4</th>
      <td>-122.25</td>
      <td>37.85</td>
      <td>52.0</td>
      <td>1627.0</td>
      <td>280.0</td>
      <td>565.0</td>
      <td>259.0</td>
      <td>3.8462</td>
      <td>342200.0</td>
      <td>NEAR BAY</td>
    </tr>
  </tbody>
</table>
</div>




```python
housing.info()
```

    <class 'pandas.core.frame.DataFrame'>
    RangeIndex: 20640 entries, 0 to 20639
    Data columns (total 10 columns):
     #   Column              Non-Null Count  Dtype  
    ---  ------              --------------  -----  
     0   longitude           20640 non-null  float64
     1   latitude            20640 non-null  float64
     2   housing_median_age  20640 non-null  float64
     3   total_rooms         20640 non-null  float64
     4   total_bedrooms      20433 non-null  float64
     5   population          20640 non-null  float64
     6   households          20640 non-null  float64
     7   median_income       20640 non-null  float64
     8   median_house_value  20640 non-null  float64
     9   ocean_proximity     20640 non-null  object 
    dtypes: float64(9), object(1)
    memory usage: 1.6+ MB



```python
housing['ocean_proximity'].value_counts()
```




    <1H OCEAN     9136
    INLAND        6551
    NEAR OCEAN    2658
    NEAR BAY      2290
    ISLAND           5
    Name: ocean_proximity, dtype: int64




```python
housing.describe()
```




<div>
<style scoped>
    .dataframe tbody tr th:only-of-type {
        vertical-align: middle;
    }

    .dataframe tbody tr th {
        vertical-align: top;
    }

    .dataframe thead th {
        text-align: right;
    }
</style>
<table border="1" class="dataframe">
  <thead>
    <tr style="text-align: right;">
      <th></th>
      <th>longitude</th>
      <th>latitude</th>
      <th>housing_median_age</th>
      <th>total_rooms</th>
      <th>total_bedrooms</th>
      <th>population</th>
      <th>households</th>
      <th>median_income</th>
      <th>median_house_value</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <th>count</th>
      <td>20640.000000</td>
      <td>20640.000000</td>
      <td>20640.000000</td>
      <td>20640.000000</td>
      <td>20433.000000</td>
      <td>20640.000000</td>
      <td>20640.000000</td>
      <td>20640.000000</td>
      <td>20640.000000</td>
    </tr>
    <tr>
      <th>mean</th>
      <td>-119.569704</td>
      <td>35.631861</td>
      <td>28.639486</td>
      <td>2635.763081</td>
      <td>537.870553</td>
      <td>1425.476744</td>
      <td>499.539680</td>
      <td>3.870671</td>
      <td>206855.816909</td>
    </tr>
    <tr>
      <th>std</th>
      <td>2.003532</td>
      <td>2.135952</td>
      <td>12.585558</td>
      <td>2181.615252</td>
      <td>421.385070</td>
      <td>1132.462122</td>
      <td>382.329753</td>
      <td>1.899822</td>
      <td>115395.615874</td>
    </tr>
    <tr>
      <th>min</th>
      <td>-124.350000</td>
      <td>32.540000</td>
      <td>1.000000</td>
      <td>2.000000</td>
      <td>1.000000</td>
      <td>3.000000</td>
      <td>1.000000</td>
      <td>0.499900</td>
      <td>14999.000000</td>
    </tr>
    <tr>
      <th>25%</th>
      <td>-121.800000</td>
      <td>33.930000</td>
      <td>18.000000</td>
      <td>1447.750000</td>
      <td>296.000000</td>
      <td>787.000000</td>
      <td>280.000000</td>
      <td>2.563400</td>
      <td>119600.000000</td>
    </tr>
    <tr>
      <th>50%</th>
      <td>-118.490000</td>
      <td>34.260000</td>
      <td>29.000000</td>
      <td>2127.000000</td>
      <td>435.000000</td>
      <td>1166.000000</td>
      <td>409.000000</td>
      <td>3.534800</td>
      <td>179700.000000</td>
    </tr>
    <tr>
      <th>75%</th>
      <td>-118.010000</td>
      <td>37.710000</td>
      <td>37.000000</td>
      <td>3148.000000</td>
      <td>647.000000</td>
      <td>1725.000000</td>
      <td>605.000000</td>
      <td>4.743250</td>
      <td>264725.000000</td>
    </tr>
    <tr>
      <th>max</th>
      <td>-114.310000</td>
      <td>41.950000</td>
      <td>52.000000</td>
      <td>39320.000000</td>
      <td>6445.000000</td>
      <td>35682.000000</td>
      <td>6082.000000</td>
      <td>15.000100</td>
      <td>500001.000000</td>
    </tr>
  </tbody>
</table>
</div>




```python
%matplotlib inline
import matplotlib.pyplot as plt
housing.hist(bins = 50, figsize = (20,15))
plt.show()
```


    
![png](output_9_0.png)
    


- 単位は何か気にすること
- 上限下限を切っている場合はその部分だけ極端に大きくなっている　→　住宅価格の中央値や築年数の中央値が今回のそれに該当する


# テストセットを作る


```python
import numpy as np

def split_tarin_test(data, test_ratio):
    shuffled_indices = np.random.permutation(len(data))#len(data)の数値までのランダムなnumpy配列を作成
    test_set_size = int(len(data) * test_ratio)
    test_indices = shuffled_indices[ : test_set_size]
    train_indices = shuffled_indices[test_set_size : ]
    return data.iloc[train_indices], data.iloc[test_indices]
```


```python
train_set, test_set = split_tarin_test(housing, 0.2)
len(train_set)
```




    16512




```python
len(test_set)
```




    4128




```python
train_set
```




<div>
<style scoped>
    .dataframe tbody tr th:only-of-type {
        vertical-align: middle;
    }

    .dataframe tbody tr th {
        vertical-align: top;
    }

    .dataframe thead th {
        text-align: right;
    }
</style>
<table border="1" class="dataframe">
  <thead>
    <tr style="text-align: right;">
      <th></th>
      <th>longitude</th>
      <th>latitude</th>
      <th>housing_median_age</th>
      <th>total_rooms</th>
      <th>total_bedrooms</th>
      <th>population</th>
      <th>households</th>
      <th>median_income</th>
      <th>median_house_value</th>
      <th>ocean_proximity</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <th>14196</th>
      <td>-117.03</td>
      <td>32.71</td>
      <td>33.0</td>
      <td>3126.0</td>
      <td>627.0</td>
      <td>2300.0</td>
      <td>623.0</td>
      <td>3.2596</td>
      <td>103000.0</td>
      <td>NEAR OCEAN</td>
    </tr>
    <tr>
      <th>8267</th>
      <td>-118.16</td>
      <td>33.77</td>
      <td>49.0</td>
      <td>3382.0</td>
      <td>787.0</td>
      <td>1314.0</td>
      <td>756.0</td>
      <td>3.8125</td>
      <td>382100.0</td>
      <td>NEAR OCEAN</td>
    </tr>
    <tr>
      <th>17445</th>
      <td>-120.48</td>
      <td>34.66</td>
      <td>4.0</td>
      <td>1897.0</td>
      <td>331.0</td>
      <td>915.0</td>
      <td>336.0</td>
      <td>4.1563</td>
      <td>172600.0</td>
      <td>NEAR OCEAN</td>
    </tr>
    <tr>
      <th>14265</th>
      <td>-117.11</td>
      <td>32.69</td>
      <td>36.0</td>
      <td>1421.0</td>
      <td>367.0</td>
      <td>1418.0</td>
      <td>355.0</td>
      <td>1.9425</td>
      <td>93400.0</td>
      <td>NEAR OCEAN</td>
    </tr>
    <tr>
      <th>2271</th>
      <td>-119.80</td>
      <td>36.78</td>
      <td>43.0</td>
      <td>2382.0</td>
      <td>431.0</td>
      <td>874.0</td>
      <td>380.0</td>
      <td>3.5542</td>
      <td>96500.0</td>
      <td>INLAND</td>
    </tr>
    <tr>
      <th>...</th>
      <td>...</td>
      <td>...</td>
      <td>...</td>
      <td>...</td>
      <td>...</td>
      <td>...</td>
      <td>...</td>
      <td>...</td>
      <td>...</td>
      <td>...</td>
    </tr>
    <tr>
      <th>11284</th>
      <td>-117.96</td>
      <td>33.78</td>
      <td>35.0</td>
      <td>1330.0</td>
      <td>201.0</td>
      <td>658.0</td>
      <td>217.0</td>
      <td>6.3700</td>
      <td>229200.0</td>
      <td>&lt;1H OCEAN</td>
    </tr>
    <tr>
      <th>11964</th>
      <td>-117.43</td>
      <td>34.02</td>
      <td>33.0</td>
      <td>3084.0</td>
      <td>570.0</td>
      <td>1753.0</td>
      <td>449.0</td>
      <td>3.0500</td>
      <td>97800.0</td>
      <td>INLAND</td>
    </tr>
    <tr>
      <th>5390</th>
      <td>-118.38</td>
      <td>34.03</td>
      <td>36.0</td>
      <td>2101.0</td>
      <td>569.0</td>
      <td>1756.0</td>
      <td>527.0</td>
      <td>2.9344</td>
      <td>222100.0</td>
      <td>&lt;1H OCEAN</td>
    </tr>
    <tr>
      <th>860</th>
      <td>-121.96</td>
      <td>37.58</td>
      <td>15.0</td>
      <td>3575.0</td>
      <td>597.0</td>
      <td>1777.0</td>
      <td>559.0</td>
      <td>5.7192</td>
      <td>283500.0</td>
      <td>&lt;1H OCEAN</td>
    </tr>
    <tr>
      <th>15795</th>
      <td>-122.42</td>
      <td>37.77</td>
      <td>52.0</td>
      <td>4226.0</td>
      <td>1315.0</td>
      <td>2619.0</td>
      <td>1242.0</td>
      <td>2.5755</td>
      <td>325000.0</td>
      <td>NEAR BAY</td>
    </tr>
  </tbody>
</table>
<p>16512 rows × 10 columns</p>
</div>



この状態だと、関数を実行するたびに異なる配列が作成されてしまい、データセット全体を見ることになる
テストセットをなんのために作ったのか分からない


```python
from sklearn.model_selection import train_test_split
train_set, test_set = train_test_split(housing, test_size = 0.2, random_state = 42)

```


```python
len(train_set)
```




    16512




```python
len(test_set)
```




    4128




```python
train_set
```




<div>
<style scoped>
    .dataframe tbody tr th:only-of-type {
        vertical-align: middle;
    }

    .dataframe tbody tr th {
        vertical-align: top;
    }

    .dataframe thead th {
        text-align: right;
    }
</style>
<table border="1" class="dataframe">
  <thead>
    <tr style="text-align: right;">
      <th></th>
      <th>longitude</th>
      <th>latitude</th>
      <th>housing_median_age</th>
      <th>total_rooms</th>
      <th>total_bedrooms</th>
      <th>population</th>
      <th>households</th>
      <th>median_income</th>
      <th>median_house_value</th>
      <th>ocean_proximity</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <th>14196</th>
      <td>-117.03</td>
      <td>32.71</td>
      <td>33.0</td>
      <td>3126.0</td>
      <td>627.0</td>
      <td>2300.0</td>
      <td>623.0</td>
      <td>3.2596</td>
      <td>103000.0</td>
      <td>NEAR OCEAN</td>
    </tr>
    <tr>
      <th>8267</th>
      <td>-118.16</td>
      <td>33.77</td>
      <td>49.0</td>
      <td>3382.0</td>
      <td>787.0</td>
      <td>1314.0</td>
      <td>756.0</td>
      <td>3.8125</td>
      <td>382100.0</td>
      <td>NEAR OCEAN</td>
    </tr>
    <tr>
      <th>17445</th>
      <td>-120.48</td>
      <td>34.66</td>
      <td>4.0</td>
      <td>1897.0</td>
      <td>331.0</td>
      <td>915.0</td>
      <td>336.0</td>
      <td>4.1563</td>
      <td>172600.0</td>
      <td>NEAR OCEAN</td>
    </tr>
    <tr>
      <th>14265</th>
      <td>-117.11</td>
      <td>32.69</td>
      <td>36.0</td>
      <td>1421.0</td>
      <td>367.0</td>
      <td>1418.0</td>
      <td>355.0</td>
      <td>1.9425</td>
      <td>93400.0</td>
      <td>NEAR OCEAN</td>
    </tr>
    <tr>
      <th>2271</th>
      <td>-119.80</td>
      <td>36.78</td>
      <td>43.0</td>
      <td>2382.0</td>
      <td>431.0</td>
      <td>874.0</td>
      <td>380.0</td>
      <td>3.5542</td>
      <td>96500.0</td>
      <td>INLAND</td>
    </tr>
    <tr>
      <th>...</th>
      <td>...</td>
      <td>...</td>
      <td>...</td>
      <td>...</td>
      <td>...</td>
      <td>...</td>
      <td>...</td>
      <td>...</td>
      <td>...</td>
      <td>...</td>
    </tr>
    <tr>
      <th>11284</th>
      <td>-117.96</td>
      <td>33.78</td>
      <td>35.0</td>
      <td>1330.0</td>
      <td>201.0</td>
      <td>658.0</td>
      <td>217.0</td>
      <td>6.3700</td>
      <td>229200.0</td>
      <td>&lt;1H OCEAN</td>
    </tr>
    <tr>
      <th>11964</th>
      <td>-117.43</td>
      <td>34.02</td>
      <td>33.0</td>
      <td>3084.0</td>
      <td>570.0</td>
      <td>1753.0</td>
      <td>449.0</td>
      <td>3.0500</td>
      <td>97800.0</td>
      <td>INLAND</td>
    </tr>
    <tr>
      <th>5390</th>
      <td>-118.38</td>
      <td>34.03</td>
      <td>36.0</td>
      <td>2101.0</td>
      <td>569.0</td>
      <td>1756.0</td>
      <td>527.0</td>
      <td>2.9344</td>
      <td>222100.0</td>
      <td>&lt;1H OCEAN</td>
    </tr>
    <tr>
      <th>860</th>
      <td>-121.96</td>
      <td>37.58</td>
      <td>15.0</td>
      <td>3575.0</td>
      <td>597.0</td>
      <td>1777.0</td>
      <td>559.0</td>
      <td>5.7192</td>
      <td>283500.0</td>
      <td>&lt;1H OCEAN</td>
    </tr>
    <tr>
      <th>15795</th>
      <td>-122.42</td>
      <td>37.77</td>
      <td>52.0</td>
      <td>4226.0</td>
      <td>1315.0</td>
      <td>2619.0</td>
      <td>1242.0</td>
      <td>2.5755</td>
      <td>325000.0</td>
      <td>NEAR BAY</td>
    </tr>
  </tbody>
</table>
<p>16512 rows × 10 columns</p>
</div>



層化抽出法


```python
housing['income_cat'] = pd.cut(housing['median_income'], 
                              bins=[0., 1.5, 3.0, 4.5, 6., np.inf],
                              labels=[1,2,3,4,5])
housing['income_cat'].hist()
```




    <AxesSubplot:>




    
![png](output_22_1.png)
    



```python
from sklearn.model_selection import  StratifiedShuffleSplit
split = StratifiedShuffleSplit(n_splits=1, test_size=0.2, random_state=42)
for train_index, test_index in split.split(housing, housing['income_cat']):
    strat_train_set = housing.loc[train_index]
    strat_test_set = housing.loc[test_index]

```


```python
strat_test_set['income_cat'].value_counts() / len(strat_test_set)
```




    3    0.350533
    2    0.318798
    4    0.176357
    5    0.114583
    1    0.039729
    Name: income_cat, dtype: float64




```python
len(strat_test_set)
```




    4128




```python
len(strat_test_set['income_cat'])
```




    4128




```python
strat_test_set
```




<div>
<style scoped>
    .dataframe tbody tr th:only-of-type {
        vertical-align: middle;
    }

    .dataframe tbody tr th {
        vertical-align: top;
    }

    .dataframe thead th {
        text-align: right;
    }
</style>
<table border="1" class="dataframe">
  <thead>
    <tr style="text-align: right;">
      <th></th>
      <th>longitude</th>
      <th>latitude</th>
      <th>housing_median_age</th>
      <th>total_rooms</th>
      <th>total_bedrooms</th>
      <th>population</th>
      <th>households</th>
      <th>median_income</th>
      <th>median_house_value</th>
      <th>ocean_proximity</th>
      <th>income_cat</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <th>5241</th>
      <td>-118.39</td>
      <td>34.12</td>
      <td>29.0</td>
      <td>6447.0</td>
      <td>1012.0</td>
      <td>2184.0</td>
      <td>960.0</td>
      <td>8.2816</td>
      <td>500001.0</td>
      <td>&lt;1H OCEAN</td>
      <td>5</td>
    </tr>
    <tr>
      <th>10970</th>
      <td>-117.86</td>
      <td>33.77</td>
      <td>39.0</td>
      <td>4159.0</td>
      <td>655.0</td>
      <td>1669.0</td>
      <td>651.0</td>
      <td>4.6111</td>
      <td>240300.0</td>
      <td>&lt;1H OCEAN</td>
      <td>4</td>
    </tr>
    <tr>
      <th>20351</th>
      <td>-119.05</td>
      <td>34.21</td>
      <td>27.0</td>
      <td>4357.0</td>
      <td>926.0</td>
      <td>2110.0</td>
      <td>876.0</td>
      <td>3.0119</td>
      <td>218200.0</td>
      <td>&lt;1H OCEAN</td>
      <td>3</td>
    </tr>
    <tr>
      <th>6568</th>
      <td>-118.15</td>
      <td>34.20</td>
      <td>52.0</td>
      <td>1786.0</td>
      <td>306.0</td>
      <td>1018.0</td>
      <td>322.0</td>
      <td>4.1518</td>
      <td>182100.0</td>
      <td>INLAND</td>
      <td>3</td>
    </tr>
    <tr>
      <th>13285</th>
      <td>-117.68</td>
      <td>34.07</td>
      <td>32.0</td>
      <td>1775.0</td>
      <td>314.0</td>
      <td>1067.0</td>
      <td>302.0</td>
      <td>4.0375</td>
      <td>121300.0</td>
      <td>INLAND</td>
      <td>3</td>
    </tr>
    <tr>
      <th>...</th>
      <td>...</td>
      <td>...</td>
      <td>...</td>
      <td>...</td>
      <td>...</td>
      <td>...</td>
      <td>...</td>
      <td>...</td>
      <td>...</td>
      <td>...</td>
      <td>...</td>
    </tr>
    <tr>
      <th>20519</th>
      <td>-121.53</td>
      <td>38.58</td>
      <td>33.0</td>
      <td>4988.0</td>
      <td>1169.0</td>
      <td>2414.0</td>
      <td>1075.0</td>
      <td>1.9728</td>
      <td>76400.0</td>
      <td>INLAND</td>
      <td>2</td>
    </tr>
    <tr>
      <th>17430</th>
      <td>-120.44</td>
      <td>34.65</td>
      <td>30.0</td>
      <td>2265.0</td>
      <td>512.0</td>
      <td>1402.0</td>
      <td>471.0</td>
      <td>1.9750</td>
      <td>134000.0</td>
      <td>NEAR OCEAN</td>
      <td>2</td>
    </tr>
    <tr>
      <th>4019</th>
      <td>-118.49</td>
      <td>34.18</td>
      <td>31.0</td>
      <td>3073.0</td>
      <td>674.0</td>
      <td>1486.0</td>
      <td>684.0</td>
      <td>4.8984</td>
      <td>311700.0</td>
      <td>&lt;1H OCEAN</td>
      <td>4</td>
    </tr>
    <tr>
      <th>12107</th>
      <td>-117.32</td>
      <td>33.99</td>
      <td>27.0</td>
      <td>5464.0</td>
      <td>850.0</td>
      <td>2400.0</td>
      <td>836.0</td>
      <td>4.7110</td>
      <td>133500.0</td>
      <td>INLAND</td>
      <td>4</td>
    </tr>
    <tr>
      <th>2398</th>
      <td>-118.91</td>
      <td>36.79</td>
      <td>19.0</td>
      <td>1616.0</td>
      <td>324.0</td>
      <td>187.0</td>
      <td>80.0</td>
      <td>3.7857</td>
      <td>78600.0</td>
      <td>INLAND</td>
      <td>3</td>
    </tr>
  </tbody>
</table>
<p>4128 rows × 11 columns</p>
</div>




```python
strat_test_set['income_cat'].value_counts()
```




    3    1447
    2    1316
    4     728
    5     473
    1     164
    Name: income_cat, dtype: int64




```python
strat_test_set['income_cat']
```




    5241     5
    10970    4
    20351    3
    6568     3
    13285    3
            ..
    20519    2
    17430    2
    4019     4
    12107    4
    2398     3
    Name: income_cat, Length: 4128, dtype: category
    Categories (5, int64): [1 < 2 < 3 < 4 < 5]




```python
housing['income_cat']
```




    0        5
    1        5
    2        5
    3        4
    4        3
            ..
    20635    2
    20636    2
    20637    2
    20638    2
    20639    2
    Name: income_cat, Length: 20640, dtype: category
    Categories (5, int64): [1 < 2 < 3 < 4 < 5]




```python

```


```python
strat_train_set
```




<div>
<style scoped>
    .dataframe tbody tr th:only-of-type {
        vertical-align: middle;
    }

    .dataframe tbody tr th {
        vertical-align: top;
    }

    .dataframe thead th {
        text-align: right;
    }
</style>
<table border="1" class="dataframe">
  <thead>
    <tr style="text-align: right;">
      <th></th>
      <th>longitude</th>
      <th>latitude</th>
      <th>housing_median_age</th>
      <th>total_rooms</th>
      <th>total_bedrooms</th>
      <th>population</th>
      <th>households</th>
      <th>median_income</th>
      <th>median_house_value</th>
      <th>ocean_proximity</th>
      <th>income_cat</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <th>17606</th>
      <td>-121.89</td>
      <td>37.29</td>
      <td>38.0</td>
      <td>1568.0</td>
      <td>351.0</td>
      <td>710.0</td>
      <td>339.0</td>
      <td>2.7042</td>
      <td>286600.0</td>
      <td>&lt;1H OCEAN</td>
      <td>2</td>
    </tr>
    <tr>
      <th>18632</th>
      <td>-121.93</td>
      <td>37.05</td>
      <td>14.0</td>
      <td>679.0</td>
      <td>108.0</td>
      <td>306.0</td>
      <td>113.0</td>
      <td>6.4214</td>
      <td>340600.0</td>
      <td>&lt;1H OCEAN</td>
      <td>5</td>
    </tr>
    <tr>
      <th>14650</th>
      <td>-117.20</td>
      <td>32.77</td>
      <td>31.0</td>
      <td>1952.0</td>
      <td>471.0</td>
      <td>936.0</td>
      <td>462.0</td>
      <td>2.8621</td>
      <td>196900.0</td>
      <td>NEAR OCEAN</td>
      <td>2</td>
    </tr>
    <tr>
      <th>3230</th>
      <td>-119.61</td>
      <td>36.31</td>
      <td>25.0</td>
      <td>1847.0</td>
      <td>371.0</td>
      <td>1460.0</td>
      <td>353.0</td>
      <td>1.8839</td>
      <td>46300.0</td>
      <td>INLAND</td>
      <td>2</td>
    </tr>
    <tr>
      <th>3555</th>
      <td>-118.59</td>
      <td>34.23</td>
      <td>17.0</td>
      <td>6592.0</td>
      <td>1525.0</td>
      <td>4459.0</td>
      <td>1463.0</td>
      <td>3.0347</td>
      <td>254500.0</td>
      <td>&lt;1H OCEAN</td>
      <td>3</td>
    </tr>
    <tr>
      <th>...</th>
      <td>...</td>
      <td>...</td>
      <td>...</td>
      <td>...</td>
      <td>...</td>
      <td>...</td>
      <td>...</td>
      <td>...</td>
      <td>...</td>
      <td>...</td>
      <td>...</td>
    </tr>
    <tr>
      <th>6563</th>
      <td>-118.13</td>
      <td>34.20</td>
      <td>46.0</td>
      <td>1271.0</td>
      <td>236.0</td>
      <td>573.0</td>
      <td>210.0</td>
      <td>4.9312</td>
      <td>240200.0</td>
      <td>INLAND</td>
      <td>4</td>
    </tr>
    <tr>
      <th>12053</th>
      <td>-117.56</td>
      <td>33.88</td>
      <td>40.0</td>
      <td>1196.0</td>
      <td>294.0</td>
      <td>1052.0</td>
      <td>258.0</td>
      <td>2.0682</td>
      <td>113000.0</td>
      <td>INLAND</td>
      <td>2</td>
    </tr>
    <tr>
      <th>13908</th>
      <td>-116.40</td>
      <td>34.09</td>
      <td>9.0</td>
      <td>4855.0</td>
      <td>872.0</td>
      <td>2098.0</td>
      <td>765.0</td>
      <td>3.2723</td>
      <td>97800.0</td>
      <td>INLAND</td>
      <td>3</td>
    </tr>
    <tr>
      <th>11159</th>
      <td>-118.01</td>
      <td>33.82</td>
      <td>31.0</td>
      <td>1960.0</td>
      <td>380.0</td>
      <td>1356.0</td>
      <td>356.0</td>
      <td>4.0625</td>
      <td>225900.0</td>
      <td>&lt;1H OCEAN</td>
      <td>3</td>
    </tr>
    <tr>
      <th>15775</th>
      <td>-122.45</td>
      <td>37.77</td>
      <td>52.0</td>
      <td>3095.0</td>
      <td>682.0</td>
      <td>1269.0</td>
      <td>639.0</td>
      <td>3.5750</td>
      <td>500001.0</td>
      <td>NEAR BAY</td>
      <td>3</td>
    </tr>
  </tbody>
</table>
<p>16512 rows × 11 columns</p>
</div>




```python

```

# データを研究、可視化して理解を深める
## 地理データの可視化


```python
housing = strat_train_set.copy()
# len(housing)
# type(housing) #DataFrame
```


```python
housing.plot(kind='scatter', x ='longitude', y='latitude', alpha=0.1)
```




    <AxesSubplot:xlabel='longitude', ylabel='latitude'>




    
![png](output_36_1.png)
    



```python
housing.plot(kind='scatter', x='longitude', y='latitude', alpha=0.4,
             s=housing['population']/100, label='population', figsize=(10, 7),
             c=housing['median_house_value'], cmap=plt.get_cmap('jet'), colorbar='True',  
             sharex=False)
plt.legend()

```




    <matplotlib.legend.Legend at 0x7f185a1b5820>




    
![png](output_37_1.png)
    


## 相関を探す


```python
corr_matrix = housing.corr()
corr_matrix
```




<div>
<style scoped>
    .dataframe tbody tr th:only-of-type {
        vertical-align: middle;
    }

    .dataframe tbody tr th {
        vertical-align: top;
    }

    .dataframe thead th {
        text-align: right;
    }
</style>
<table border="1" class="dataframe">
  <thead>
    <tr style="text-align: right;">
      <th></th>
      <th>longitude</th>
      <th>latitude</th>
      <th>housing_median_age</th>
      <th>total_rooms</th>
      <th>total_bedrooms</th>
      <th>population</th>
      <th>households</th>
      <th>median_income</th>
      <th>median_house_value</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <th>longitude</th>
      <td>1.000000</td>
      <td>-0.924478</td>
      <td>-0.105848</td>
      <td>0.048871</td>
      <td>0.076598</td>
      <td>0.108030</td>
      <td>0.063070</td>
      <td>-0.019583</td>
      <td>-0.047432</td>
    </tr>
    <tr>
      <th>latitude</th>
      <td>-0.924478</td>
      <td>1.000000</td>
      <td>0.005766</td>
      <td>-0.039184</td>
      <td>-0.072419</td>
      <td>-0.115222</td>
      <td>-0.077647</td>
      <td>-0.075205</td>
      <td>-0.142724</td>
    </tr>
    <tr>
      <th>housing_median_age</th>
      <td>-0.105848</td>
      <td>0.005766</td>
      <td>1.000000</td>
      <td>-0.364509</td>
      <td>-0.325047</td>
      <td>-0.298710</td>
      <td>-0.306428</td>
      <td>-0.111360</td>
      <td>0.114110</td>
    </tr>
    <tr>
      <th>total_rooms</th>
      <td>0.048871</td>
      <td>-0.039184</td>
      <td>-0.364509</td>
      <td>1.000000</td>
      <td>0.929379</td>
      <td>0.855109</td>
      <td>0.918392</td>
      <td>0.200087</td>
      <td>0.135097</td>
    </tr>
    <tr>
      <th>total_bedrooms</th>
      <td>0.076598</td>
      <td>-0.072419</td>
      <td>-0.325047</td>
      <td>0.929379</td>
      <td>1.000000</td>
      <td>0.876320</td>
      <td>0.980170</td>
      <td>-0.009740</td>
      <td>0.047689</td>
    </tr>
    <tr>
      <th>population</th>
      <td>0.108030</td>
      <td>-0.115222</td>
      <td>-0.298710</td>
      <td>0.855109</td>
      <td>0.876320</td>
      <td>1.000000</td>
      <td>0.904637</td>
      <td>0.002380</td>
      <td>-0.026920</td>
    </tr>
    <tr>
      <th>households</th>
      <td>0.063070</td>
      <td>-0.077647</td>
      <td>-0.306428</td>
      <td>0.918392</td>
      <td>0.980170</td>
      <td>0.904637</td>
      <td>1.000000</td>
      <td>0.010781</td>
      <td>0.064506</td>
    </tr>
    <tr>
      <th>median_income</th>
      <td>-0.019583</td>
      <td>-0.075205</td>
      <td>-0.111360</td>
      <td>0.200087</td>
      <td>-0.009740</td>
      <td>0.002380</td>
      <td>0.010781</td>
      <td>1.000000</td>
      <td>0.687160</td>
    </tr>
    <tr>
      <th>median_house_value</th>
      <td>-0.047432</td>
      <td>-0.142724</td>
      <td>0.114110</td>
      <td>0.135097</td>
      <td>0.047689</td>
      <td>-0.026920</td>
      <td>0.064506</td>
      <td>0.687160</td>
      <td>1.000000</td>
    </tr>
  </tbody>
</table>
</div>




```python
corr_matrix['median_house_value'].sort_values(ascending=False)
```




    median_house_value    1.000000
    median_income         0.687160
    total_rooms           0.135097
    housing_median_age    0.114110
    households            0.064506
    total_bedrooms        0.047689
    population           -0.026920
    longitude            -0.047432
    latitude             -0.142724
    Name: median_house_value, dtype: float64




```python
from pandas.plotting import scatter_matrix

attributes = ['median_house_value', 'median_income', 'total_rooms', 'housing_median_age']
scatter_matrix(frame=housing[attributes], figsize=(12, 8))
```




    array([[<AxesSubplot:xlabel='median_house_value', ylabel='median_house_value'>,
            <AxesSubplot:xlabel='median_income', ylabel='median_house_value'>,
            <AxesSubplot:xlabel='total_rooms', ylabel='median_house_value'>,
            <AxesSubplot:xlabel='housing_median_age', ylabel='median_house_value'>],
           [<AxesSubplot:xlabel='median_house_value', ylabel='median_income'>,
            <AxesSubplot:xlabel='median_income', ylabel='median_income'>,
            <AxesSubplot:xlabel='total_rooms', ylabel='median_income'>,
            <AxesSubplot:xlabel='housing_median_age', ylabel='median_income'>],
           [<AxesSubplot:xlabel='median_house_value', ylabel='total_rooms'>,
            <AxesSubplot:xlabel='median_income', ylabel='total_rooms'>,
            <AxesSubplot:xlabel='total_rooms', ylabel='total_rooms'>,
            <AxesSubplot:xlabel='housing_median_age', ylabel='total_rooms'>],
           [<AxesSubplot:xlabel='median_house_value', ylabel='housing_median_age'>,
            <AxesSubplot:xlabel='median_income', ylabel='housing_median_age'>,
            <AxesSubplot:xlabel='total_rooms', ylabel='housing_median_age'>,
            <AxesSubplot:xlabel='housing_median_age', ylabel='housing_median_age'>]],
          dtype=object)




    
![png](output_41_1.png)
    



```python
housing.plot(kind='scatter', x='median_income', y='median_house_value', alpha=0.1)
```




    <AxesSubplot:xlabel='median_income', ylabel='median_house_value'>




    
![png](output_42_1.png)
    


## 属性の組み合わせを試してみる　特徴量エンジニアリング


```python
# 新しい属性を作ってみる
housing['room_per_household'] = housing['total_rooms']/housing['households']
housing['bedrooms_per_room'] = housing['total_bedrooms']/housing['total_rooms']
housing['population_per_household'] = housing['population']/housing['households']
```


```python
#相関係数を計算
corr_matrix = housing.corr()
corr_matrix['median_house_value'].sort_values(ascending=False)
```




    median_house_value          1.000000
    median_income               0.687160
    room_per_household          0.146285
    total_rooms                 0.135097
    housing_median_age          0.114110
    households                  0.064506
    total_bedrooms              0.047689
    population_per_household   -0.021985
    population                 -0.026920
    longitude                  -0.047432
    latitude                   -0.142724
    bedrooms_per_room          -0.259984
    Name: median_house_value, dtype: float64



# 機械学習アルゴリズムが処理しやすいようにデータを準備する
## データのクリーニング


```python
housing = strat_train_set.drop('median_house_value', axis=1)
housing_labels = strat_train_set['median_house_value'].copy()
```


```python
housing
```




<div>
<style scoped>
    .dataframe tbody tr th:only-of-type {
        vertical-align: middle;
    }

    .dataframe tbody tr th {
        vertical-align: top;
    }

    .dataframe thead th {
        text-align: right;
    }
</style>
<table border="1" class="dataframe">
  <thead>
    <tr style="text-align: right;">
      <th></th>
      <th>longitude</th>
      <th>latitude</th>
      <th>housing_median_age</th>
      <th>total_rooms</th>
      <th>total_bedrooms</th>
      <th>population</th>
      <th>households</th>
      <th>median_income</th>
      <th>ocean_proximity</th>
      <th>income_cat</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <th>17606</th>
      <td>-121.89</td>
      <td>37.29</td>
      <td>38.0</td>
      <td>1568.0</td>
      <td>351.0</td>
      <td>710.0</td>
      <td>339.0</td>
      <td>2.7042</td>
      <td>&lt;1H OCEAN</td>
      <td>2</td>
    </tr>
    <tr>
      <th>18632</th>
      <td>-121.93</td>
      <td>37.05</td>
      <td>14.0</td>
      <td>679.0</td>
      <td>108.0</td>
      <td>306.0</td>
      <td>113.0</td>
      <td>6.4214</td>
      <td>&lt;1H OCEAN</td>
      <td>5</td>
    </tr>
    <tr>
      <th>14650</th>
      <td>-117.20</td>
      <td>32.77</td>
      <td>31.0</td>
      <td>1952.0</td>
      <td>471.0</td>
      <td>936.0</td>
      <td>462.0</td>
      <td>2.8621</td>
      <td>NEAR OCEAN</td>
      <td>2</td>
    </tr>
    <tr>
      <th>3230</th>
      <td>-119.61</td>
      <td>36.31</td>
      <td>25.0</td>
      <td>1847.0</td>
      <td>371.0</td>
      <td>1460.0</td>
      <td>353.0</td>
      <td>1.8839</td>
      <td>INLAND</td>
      <td>2</td>
    </tr>
    <tr>
      <th>3555</th>
      <td>-118.59</td>
      <td>34.23</td>
      <td>17.0</td>
      <td>6592.0</td>
      <td>1525.0</td>
      <td>4459.0</td>
      <td>1463.0</td>
      <td>3.0347</td>
      <td>&lt;1H OCEAN</td>
      <td>3</td>
    </tr>
    <tr>
      <th>...</th>
      <td>...</td>
      <td>...</td>
      <td>...</td>
      <td>...</td>
      <td>...</td>
      <td>...</td>
      <td>...</td>
      <td>...</td>
      <td>...</td>
      <td>...</td>
    </tr>
    <tr>
      <th>6563</th>
      <td>-118.13</td>
      <td>34.20</td>
      <td>46.0</td>
      <td>1271.0</td>
      <td>236.0</td>
      <td>573.0</td>
      <td>210.0</td>
      <td>4.9312</td>
      <td>INLAND</td>
      <td>4</td>
    </tr>
    <tr>
      <th>12053</th>
      <td>-117.56</td>
      <td>33.88</td>
      <td>40.0</td>
      <td>1196.0</td>
      <td>294.0</td>
      <td>1052.0</td>
      <td>258.0</td>
      <td>2.0682</td>
      <td>INLAND</td>
      <td>2</td>
    </tr>
    <tr>
      <th>13908</th>
      <td>-116.40</td>
      <td>34.09</td>
      <td>9.0</td>
      <td>4855.0</td>
      <td>872.0</td>
      <td>2098.0</td>
      <td>765.0</td>
      <td>3.2723</td>
      <td>INLAND</td>
      <td>3</td>
    </tr>
    <tr>
      <th>11159</th>
      <td>-118.01</td>
      <td>33.82</td>
      <td>31.0</td>
      <td>1960.0</td>
      <td>380.0</td>
      <td>1356.0</td>
      <td>356.0</td>
      <td>4.0625</td>
      <td>&lt;1H OCEAN</td>
      <td>3</td>
    </tr>
    <tr>
      <th>15775</th>
      <td>-122.45</td>
      <td>37.77</td>
      <td>52.0</td>
      <td>3095.0</td>
      <td>682.0</td>
      <td>1269.0</td>
      <td>639.0</td>
      <td>3.5750</td>
      <td>NEAR BAY</td>
      <td>3</td>
    </tr>
  </tbody>
</table>
<p>16512 rows × 10 columns</p>
</div>




```python
housing_labels
```




    17606    286600.0
    18632    340600.0
    14650    196900.0
    3230      46300.0
    3555     254500.0
               ...   
    6563     240200.0
    12053    113000.0
    13908     97800.0
    11159    225900.0
    15775    500001.0
    Name: median_house_value, Length: 16512, dtype: float64




```python
housing.info()
```

    <class 'pandas.core.frame.DataFrame'>
    Int64Index: 16512 entries, 17606 to 15775
    Data columns (total 10 columns):
     #   Column              Non-Null Count  Dtype   
    ---  ------              --------------  -----   
     0   longitude           16512 non-null  float64 
     1   latitude            16512 non-null  float64 
     2   housing_median_age  16512 non-null  float64 
     3   total_rooms         16512 non-null  float64 
     4   total_bedrooms      16354 non-null  float64 
     5   population          16512 non-null  float64 
     6   households          16512 non-null  float64 
     7   median_income       16512 non-null  float64 
     8   ocean_proximity     16512 non-null  object  
     9   income_cat          16512 non-null  category
    dtypes: category(1), float64(8), object(1)
    memory usage: 1.3+ MB



```python
housing.isnull().any()
```




    longitude             False
    latitude              False
    housing_median_age    False
    total_rooms           False
    total_bedrooms         True
    population            False
    households            False
    median_income         False
    ocean_proximity       False
    income_cat            False
    dtype: bool




```python
median = housing['total_bedrooms'].median()
median
# housing['total_bedrooms'].head()
```




    433.0




```python
housing['ocean_proximity']#object
```




    17606     <1H OCEAN
    18632     <1H OCEAN
    14650    NEAR OCEAN
    3230         INLAND
    3555      <1H OCEAN
                ...    
    6563         INLAND
    12053        INLAND
    13908        INLAND
    11159     <1H OCEAN
    15775      NEAR BAY
    Name: ocean_proximity, Length: 16512, dtype: object




```python
from sklearn.impute import SimpleImputer
imputer = SimpleImputer(strategy='median')# 欠損値を各属性の中央値で保管するインスタンスを作成
```


```python
housing_num = housing.drop('ocean_proximity', axis=1)#テキスト属性を取り除いて、数値だけのhousingデータを作成
```


```python
imputer.fit(housing_num) # fitメソッドで訓練データhousing_numに、imputerインスタンスを適合させられる
```




    SimpleImputer(strategy='median')




```python
# 中央値が正しく格納されているか確認
imputer.statistics_ # statisticsインスタンス変数に中央値が格納している
```




    array([-118.51  ,   34.26  ,   29.    , 2119.5   ,  433.    , 1164.    ,
            408.    ,    3.5409,    3.    ])




```python
housing_num.median().values
```




    array([-118.51  ,   34.26  ,   29.    , 2119.5   ,  433.    , 1164.    ,
            408.    ,    3.5409])




```python
X = imputer.transform(housing_num) # housing_numの欠損値をすべて中央値で補完
type(X)
print(X)
```

    [[-121.89     37.29     38.     ...  339.        2.7042    2.    ]
     [-121.93     37.05     14.     ...  113.        6.4214    5.    ]
     [-117.2      32.77     31.     ...  462.        2.8621    2.    ]
     ...
     [-116.4      34.09      9.     ...  765.        3.2723    3.    ]
     [-118.01     33.82     31.     ...  356.        4.0625    3.    ]
     [-122.45     37.77     52.     ...  639.        3.575     3.    ]]



```python
housing_tr = pd.DataFrame(data=X, columns=housing_num.columns, index=housing_num.index)
housing_tr
```




<div>
<style scoped>
    .dataframe tbody tr th:only-of-type {
        vertical-align: middle;
    }

    .dataframe tbody tr th {
        vertical-align: top;
    }

    .dataframe thead th {
        text-align: right;
    }
</style>
<table border="1" class="dataframe">
  <thead>
    <tr style="text-align: right;">
      <th></th>
      <th>longitude</th>
      <th>latitude</th>
      <th>housing_median_age</th>
      <th>total_rooms</th>
      <th>total_bedrooms</th>
      <th>population</th>
      <th>households</th>
      <th>median_income</th>
      <th>income_cat</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <th>17606</th>
      <td>-121.89</td>
      <td>37.29</td>
      <td>38.0</td>
      <td>1568.0</td>
      <td>351.0</td>
      <td>710.0</td>
      <td>339.0</td>
      <td>2.7042</td>
      <td>2.0</td>
    </tr>
    <tr>
      <th>18632</th>
      <td>-121.93</td>
      <td>37.05</td>
      <td>14.0</td>
      <td>679.0</td>
      <td>108.0</td>
      <td>306.0</td>
      <td>113.0</td>
      <td>6.4214</td>
      <td>5.0</td>
    </tr>
    <tr>
      <th>14650</th>
      <td>-117.20</td>
      <td>32.77</td>
      <td>31.0</td>
      <td>1952.0</td>
      <td>471.0</td>
      <td>936.0</td>
      <td>462.0</td>
      <td>2.8621</td>
      <td>2.0</td>
    </tr>
    <tr>
      <th>3230</th>
      <td>-119.61</td>
      <td>36.31</td>
      <td>25.0</td>
      <td>1847.0</td>
      <td>371.0</td>
      <td>1460.0</td>
      <td>353.0</td>
      <td>1.8839</td>
      <td>2.0</td>
    </tr>
    <tr>
      <th>3555</th>
      <td>-118.59</td>
      <td>34.23</td>
      <td>17.0</td>
      <td>6592.0</td>
      <td>1525.0</td>
      <td>4459.0</td>
      <td>1463.0</td>
      <td>3.0347</td>
      <td>3.0</td>
    </tr>
    <tr>
      <th>...</th>
      <td>...</td>
      <td>...</td>
      <td>...</td>
      <td>...</td>
      <td>...</td>
      <td>...</td>
      <td>...</td>
      <td>...</td>
      <td>...</td>
    </tr>
    <tr>
      <th>6563</th>
      <td>-118.13</td>
      <td>34.20</td>
      <td>46.0</td>
      <td>1271.0</td>
      <td>236.0</td>
      <td>573.0</td>
      <td>210.0</td>
      <td>4.9312</td>
      <td>4.0</td>
    </tr>
    <tr>
      <th>12053</th>
      <td>-117.56</td>
      <td>33.88</td>
      <td>40.0</td>
      <td>1196.0</td>
      <td>294.0</td>
      <td>1052.0</td>
      <td>258.0</td>
      <td>2.0682</td>
      <td>2.0</td>
    </tr>
    <tr>
      <th>13908</th>
      <td>-116.40</td>
      <td>34.09</td>
      <td>9.0</td>
      <td>4855.0</td>
      <td>872.0</td>
      <td>2098.0</td>
      <td>765.0</td>
      <td>3.2723</td>
      <td>3.0</td>
    </tr>
    <tr>
      <th>11159</th>
      <td>-118.01</td>
      <td>33.82</td>
      <td>31.0</td>
      <td>1960.0</td>
      <td>380.0</td>
      <td>1356.0</td>
      <td>356.0</td>
      <td>4.0625</td>
      <td>3.0</td>
    </tr>
    <tr>
      <th>15775</th>
      <td>-122.45</td>
      <td>37.77</td>
      <td>52.0</td>
      <td>3095.0</td>
      <td>682.0</td>
      <td>1269.0</td>
      <td>639.0</td>
      <td>3.5750</td>
      <td>3.0</td>
    </tr>
  </tbody>
</table>
<p>16512 rows × 9 columns</p>
</div>




```python
housing_tr.isnull().any() #　欠損値が補完されていることが分かる
```




    longitude             False
    latitude              False
    housing_median_age    False
    total_rooms           False
    total_bedrooms        False
    population            False
    households            False
    median_income         False
    income_cat            False
    dtype: bool



## テキスト/カテゴリ属性の処理


```python
housing
```




<div>
<style scoped>
    .dataframe tbody tr th:only-of-type {
        vertical-align: middle;
    }

    .dataframe tbody tr th {
        vertical-align: top;
    }

    .dataframe thead th {
        text-align: right;
    }
</style>
<table border="1" class="dataframe">
  <thead>
    <tr style="text-align: right;">
      <th></th>
      <th>longitude</th>
      <th>latitude</th>
      <th>housing_median_age</th>
      <th>total_rooms</th>
      <th>total_bedrooms</th>
      <th>population</th>
      <th>households</th>
      <th>median_income</th>
      <th>ocean_proximity</th>
      <th>income_cat</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <th>17606</th>
      <td>-121.89</td>
      <td>37.29</td>
      <td>38.0</td>
      <td>1568.0</td>
      <td>351.0</td>
      <td>710.0</td>
      <td>339.0</td>
      <td>2.7042</td>
      <td>&lt;1H OCEAN</td>
      <td>2</td>
    </tr>
    <tr>
      <th>18632</th>
      <td>-121.93</td>
      <td>37.05</td>
      <td>14.0</td>
      <td>679.0</td>
      <td>108.0</td>
      <td>306.0</td>
      <td>113.0</td>
      <td>6.4214</td>
      <td>&lt;1H OCEAN</td>
      <td>5</td>
    </tr>
    <tr>
      <th>14650</th>
      <td>-117.20</td>
      <td>32.77</td>
      <td>31.0</td>
      <td>1952.0</td>
      <td>471.0</td>
      <td>936.0</td>
      <td>462.0</td>
      <td>2.8621</td>
      <td>NEAR OCEAN</td>
      <td>2</td>
    </tr>
    <tr>
      <th>3230</th>
      <td>-119.61</td>
      <td>36.31</td>
      <td>25.0</td>
      <td>1847.0</td>
      <td>371.0</td>
      <td>1460.0</td>
      <td>353.0</td>
      <td>1.8839</td>
      <td>INLAND</td>
      <td>2</td>
    </tr>
    <tr>
      <th>3555</th>
      <td>-118.59</td>
      <td>34.23</td>
      <td>17.0</td>
      <td>6592.0</td>
      <td>1525.0</td>
      <td>4459.0</td>
      <td>1463.0</td>
      <td>3.0347</td>
      <td>&lt;1H OCEAN</td>
      <td>3</td>
    </tr>
    <tr>
      <th>...</th>
      <td>...</td>
      <td>...</td>
      <td>...</td>
      <td>...</td>
      <td>...</td>
      <td>...</td>
      <td>...</td>
      <td>...</td>
      <td>...</td>
      <td>...</td>
    </tr>
    <tr>
      <th>6563</th>
      <td>-118.13</td>
      <td>34.20</td>
      <td>46.0</td>
      <td>1271.0</td>
      <td>236.0</td>
      <td>573.0</td>
      <td>210.0</td>
      <td>4.9312</td>
      <td>INLAND</td>
      <td>4</td>
    </tr>
    <tr>
      <th>12053</th>
      <td>-117.56</td>
      <td>33.88</td>
      <td>40.0</td>
      <td>1196.0</td>
      <td>294.0</td>
      <td>1052.0</td>
      <td>258.0</td>
      <td>2.0682</td>
      <td>INLAND</td>
      <td>2</td>
    </tr>
    <tr>
      <th>13908</th>
      <td>-116.40</td>
      <td>34.09</td>
      <td>9.0</td>
      <td>4855.0</td>
      <td>872.0</td>
      <td>2098.0</td>
      <td>765.0</td>
      <td>3.2723</td>
      <td>INLAND</td>
      <td>3</td>
    </tr>
    <tr>
      <th>11159</th>
      <td>-118.01</td>
      <td>33.82</td>
      <td>31.0</td>
      <td>1960.0</td>
      <td>380.0</td>
      <td>1356.0</td>
      <td>356.0</td>
      <td>4.0625</td>
      <td>&lt;1H OCEAN</td>
      <td>3</td>
    </tr>
    <tr>
      <th>15775</th>
      <td>-122.45</td>
      <td>37.77</td>
      <td>52.0</td>
      <td>3095.0</td>
      <td>682.0</td>
      <td>1269.0</td>
      <td>639.0</td>
      <td>3.5750</td>
      <td>NEAR BAY</td>
      <td>3</td>
    </tr>
  </tbody>
</table>
<p>16512 rows × 10 columns</p>
</div>




```python
housing.isnull().any()
```




    longitude             False
    latitude              False
    housing_median_age    False
    total_rooms           False
    total_bedrooms         True
    population            False
    households            False
    median_income         False
    ocean_proximity       False
    income_cat            False
    dtype: bool




```python
housing_cat = housing['ocean_proximity']
housing_cat.head(10)
```




    17606     <1H OCEAN
    18632     <1H OCEAN
    14650    NEAR OCEAN
    3230         INLAND
    3555      <1H OCEAN
    19480        INLAND
    8879      <1H OCEAN
    13685        INLAND
    4937      <1H OCEAN
    4861      <1H OCEAN
    Name: ocean_proximity, dtype: object




```python
# テキストラベルを数値に変換　目的：機械学習アルゴリズムは数値属性のほうが操作しやすいため

from sklearn.preprocessing import OrdinalEncoder

ordinal_encoder = OrdinalEncoder()
housing_cat_encoded = ordinal_encoder.fit_transform(housing_cat)
```


    ---------------------------------------------------------------------------

    ValueError                                Traceback (most recent call last)

    <ipython-input-162-bf255ca0d523> in <module>
          4 
          5 ordinal_encoder = OrdinalEncoder()
    ----> 6 housing_cat_encoded = ordinal_encoder.fit_transform(housing_cat)
    

    ~/anaconda3/lib/python3.8/site-packages/sklearn/base.py in fit_transform(self, X, y, **fit_params)
        697         if y is None:
        698             # fit method of arity 1 (unsupervised transformation)
    --> 699             return self.fit(X, **fit_params).transform(X)
        700         else:
        701             # fit method of arity 2 (supervised transformation)


    ~/anaconda3/lib/python3.8/site-packages/sklearn/preprocessing/_encoders.py in fit(self, X, y)
        759                             f"got {self.unknown_value}.")
        760 
    --> 761         self._fit(X)
        762 
        763         if self.handle_unknown == 'use_encoded_value':


    ~/anaconda3/lib/python3.8/site-packages/sklearn/preprocessing/_encoders.py in _fit(self, X, handle_unknown, force_all_finite)
         75 
         76     def _fit(self, X, handle_unknown='error', force_all_finite=True):
    ---> 77         X_list, n_samples, n_features = self._check_X(
         78             X, force_all_finite=force_all_finite)
         79 


    ~/anaconda3/lib/python3.8/site-packages/sklearn/preprocessing/_encoders.py in _check_X(self, X, force_all_finite)
         42         if not (hasattr(X, 'iloc') and getattr(X, 'ndim', 0) == 2):
         43             # if not a dataframe, do normal check_array validation
    ---> 44             X_temp = check_array(X, dtype=None,
         45                                  force_all_finite=force_all_finite)
         46             if (not hasattr(X, 'dtype')


    ~/anaconda3/lib/python3.8/site-packages/sklearn/utils/validation.py in inner_f(*args, **kwargs)
         61             extra_args = len(args) - len(all_args)
         62             if extra_args <= 0:
    ---> 63                 return f(*args, **kwargs)
         64 
         65             # extra_args > 0


    ~/anaconda3/lib/python3.8/site-packages/sklearn/utils/validation.py in check_array(array, accept_sparse, accept_large_sparse, dtype, order, copy, force_all_finite, ensure_2d, allow_nd, ensure_min_samples, ensure_min_features, estimator)
        635             # If input is 1D raise error
        636             if array.ndim == 1:
    --> 637                 raise ValueError(
        638                     "Expected 2D array, got 1D array instead:\narray={}.\n"
        639                     "Reshape your data either using array.reshape(-1, 1) if "


    ValueError: Expected 2D array, got 1D array instead:
    array=['<1H OCEAN' '<1H OCEAN' 'NEAR OCEAN' ... 'INLAND' '<1H OCEAN' 'NEAR BAY'].
    Reshape your data either using array.reshape(-1, 1) if your data has a single feature or array.reshape(1, -1) if it contains a single sample.



```python

```