{ "cells": [ { "attachments": {}, "cell_type": "markdown", "id": "4c9be1ce", "metadata": {}, "source": [ "Basic Statistics\n", "================\n", "\n", "This Notebook shows us how to use the `pandas` to find\n", "basic quantitative descriptions of our data\n", "\n", "Topics in this Notebook:\n", "\n", "- dropping columns\n", "- minimum and maximum ranges\n", "- averages\n", "- counts\n", "- sorting data\n", "- correlations with `corr()`\n", "- `describe()`" ] }, { "cell_type": "code", "execution_count": 1, "id": "98a332e1", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
dbndistrictboroschool_nametotal_enrollmentasian_pctblack_pcthispanic_pctwhite_pctswd_pctell_pctpoverty_pct
401M0151ManhattanP.S. 015 Roberto Clemente1930.1350.2750.5280.0570.2230.1090.819
901M0191ManhattanP.S. 019 Asher Levy2120.0610.1930.6130.0800.3920.0420.712
1401M0201ManhattanP.S. 020 Anna Silver4120.2480.1330.5220.0730.2180.1190.709
1901M0341ManhattanP.S. 034 Franklin D. Roosevelt2730.0260.3810.5570.0290.3920.0620.960
2401M0631ManhattanThe STAR Academy - P.S.632080.0290.1920.6350.0910.2790.0140.769
\n", "
" ], "text/plain": [ " dbn district boro school_name \\\n", "4 01M015 1 Manhattan P.S. 015 Roberto Clemente \n", "9 01M019 1 Manhattan P.S. 019 Asher Levy \n", "14 01M020 1 Manhattan P.S. 020 Anna Silver \n", "19 01M034 1 Manhattan P.S. 034 Franklin D. Roosevelt \n", "24 01M063 1 Manhattan The STAR Academy - P.S.63 \n", "\n", " total_enrollment asian_pct black_pct hispanic_pct white_pct swd_pct \\\n", "4 193 0.135 0.275 0.528 0.057 0.223 \n", "9 212 0.061 0.193 0.613 0.080 0.392 \n", "14 412 0.248 0.133 0.522 0.073 0.218 \n", "19 273 0.026 0.381 0.557 0.029 0.392 \n", "24 208 0.029 0.192 0.635 0.091 0.279 \n", "\n", " ell_pct poverty_pct \n", "4 0.109 0.819 \n", "9 0.042 0.712 \n", "14 0.119 0.709 \n", "19 0.062 0.960 \n", "24 0.014 0.769 " ] }, "execution_count": 1, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# import schools from the nycschool package\n", "from nycschools import schools\n", "\n", "# load the demographic data into a `DataFrame` called df\n", "df = schools.load_school_demographics()\n", "\n", "# let's just use one year of data\n", "df = df[df.ay == 2020]\n", "\n", "# use a subset of columns for this notebook\n", "cols = [\n", " 'dbn',\n", " 'district',\n", " 'boro',\n", " 'school_name',\n", " 'total_enrollment',\n", " 'asian_pct',\n", " 'black_pct',\n", " 'hispanic_pct',\n", " 'white_pct',\n", " 'swd_pct',\n", " 'ell_pct',\n", " 'poverty_pct'\n", "]\n", "df = df[cols]\n", "\n", "\n", "df.head()" ] }, { "cell_type": "code", "execution_count": 2, "id": "dcec0f0c", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
dbndistrictboroschool_nametotal_enrollmentasian_pctblack_pcthispanic_pctwhite_pctswd_pctell_pctpoverty_pct
372313K43013BrooklynBrooklyn Technical High School59210.5990.0560.0660.2370.0160.0010.590
531020K49020BrooklynFort Hamilton High School46780.2900.0250.3170.3470.1530.1620.706
668626Q43026QueensFrancis Lewis High School44240.5770.0470.2100.1500.1500.1150.690
580222K40522BrooklynMidwood High School41090.3540.2550.1330.2250.1380.0360.723
580722K42522BrooklynJames Madison High School38510.2090.1280.1750.4720.1540.1210.758
727928Q44028QueensForest Hills High School37750.2470.0650.3810.2720.1640.0930.702
819931R45531Staten IslandTottenville High School37260.0720.0140.1320.7610.2270.0230.416
555721K52521BrooklynEdward R. Murrow High School36910.2740.1760.1930.3080.1780.1250.670
530020K44520BrooklynNew Utrecht High School35720.3910.0300.3360.2340.1650.2180.796
668126Q41526QueensBenjamin N. Cardozo High School34050.4230.2140.2230.1200.1450.0510.668
\n", "
" ], "text/plain": [ " dbn district boro school_name \\\n", "3723 13K430 13 Brooklyn Brooklyn Technical High School \n", "5310 20K490 20 Brooklyn Fort Hamilton High School \n", "6686 26Q430 26 Queens Francis Lewis High School \n", "5802 22K405 22 Brooklyn Midwood High School \n", "5807 22K425 22 Brooklyn James Madison High School \n", "7279 28Q440 28 Queens Forest Hills High School \n", "8199 31R455 31 Staten Island Tottenville High School \n", "5557 21K525 21 Brooklyn Edward R. Murrow High School \n", "5300 20K445 20 Brooklyn New Utrecht High School \n", "6681 26Q415 26 Queens Benjamin N. Cardozo High School \n", "\n", " total_enrollment asian_pct black_pct hispanic_pct white_pct \\\n", "3723 5921 0.599 0.056 0.066 0.237 \n", "5310 4678 0.290 0.025 0.317 0.347 \n", "6686 4424 0.577 0.047 0.210 0.150 \n", "5802 4109 0.354 0.255 0.133 0.225 \n", "5807 3851 0.209 0.128 0.175 0.472 \n", "7279 3775 0.247 0.065 0.381 0.272 \n", "8199 3726 0.072 0.014 0.132 0.761 \n", "5557 3691 0.274 0.176 0.193 0.308 \n", "5300 3572 0.391 0.030 0.336 0.234 \n", "6681 3405 0.423 0.214 0.223 0.120 \n", "\n", " swd_pct ell_pct poverty_pct \n", "3723 0.016 0.001 0.590 \n", "5310 0.153 0.162 0.706 \n", "6686 0.150 0.115 0.690 \n", "5802 0.138 0.036 0.723 \n", "5807 0.154 0.121 0.758 \n", "7279 0.164 0.093 0.702 \n", "8199 0.227 0.023 0.416 \n", "5557 0.178 0.125 0.670 \n", "5300 0.165 0.218 0.796 \n", "6681 0.145 0.051 0.668 " ] }, "execution_count": 2, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# sort the data and show just the 10 largest schools\n", "# sort in descending order (biggest --> smallest)\n", "data = df.sort_values(by=\"total_enrollment\", ascending=False)\n", "\n", "# show the first 10 rows\n", "data[:10]" ] }, { "cell_type": "code", "execution_count": 3, "id": "2386fe42", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "The largest school: 5921\n", "The smallest school: 7\n", "Avg (mean) school size: 560.79296875\n", "Avg (median) school size: 460.5\n", "Avg (mode, can return multiple values) school size: [479, 714]\n" ] } ], "source": [ "# get just the total_enrollment column, called a Series in pandas\n", "enrollment = df[\"total_enrollment\"]\n", "print(\"The largest school:\", enrollment.max())\n", "print(\"The smallest school:\", enrollment.min())\n", "\n", "print(\"Avg (mean) school size:\", enrollment.mean())\n", "print(\"Avg (median) school size:\", enrollment.median())\n", "print(\"Avg (mode, can return multiple values) school size:\", list(enrollment.mode()))" ] }, { "cell_type": "code", "execution_count": 4, "id": "2cf28566", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
districttotal_enrollmentasian_pctblack_pcthispanic_pctwhite_pctswd_pctell_pctpoverty_pct
count2048.000002048.0000002048.0000002048.0000002048.0000002048.0000002048.0000002048.0000002048.000000
mean27.62207560.7929690.1177100.3026440.4317210.1186020.2376410.1412120.766560
std26.68098461.0145100.1668040.2563750.2461180.1680410.1607800.1431270.194297
min1.000007.0000000.0000000.0000000.0150000.0000000.0000000.0000000.050000
25%10.00000304.0000000.0150000.0814740.2060000.0160000.1630000.0490000.711000
50%19.00000460.5000000.0460000.2450000.4050000.0360000.2080000.1000000.827000
75%30.00000663.2500000.1480000.4722500.6312500.1452500.2610000.1872500.903000
max84.000005921.0000000.9260000.9350001.0000000.9450001.0000001.0000000.960000
\n", "
" ], "text/plain": [ " district total_enrollment asian_pct black_pct hispanic_pct \\\n", "count 2048.00000 2048.000000 2048.000000 2048.000000 2048.000000 \n", "mean 27.62207 560.792969 0.117710 0.302644 0.431721 \n", "std 26.68098 461.014510 0.166804 0.256375 0.246118 \n", "min 1.00000 7.000000 0.000000 0.000000 0.015000 \n", "25% 10.00000 304.000000 0.015000 0.081474 0.206000 \n", "50% 19.00000 460.500000 0.046000 0.245000 0.405000 \n", "75% 30.00000 663.250000 0.148000 0.472250 0.631250 \n", "max 84.00000 5921.000000 0.926000 0.935000 1.000000 \n", "\n", " white_pct swd_pct ell_pct poverty_pct \n", "count 2048.000000 2048.000000 2048.000000 2048.000000 \n", "mean 0.118602 0.237641 0.141212 0.766560 \n", "std 0.168041 0.160780 0.143127 0.194297 \n", "min 0.000000 0.000000 0.000000 0.050000 \n", "25% 0.016000 0.163000 0.049000 0.711000 \n", "50% 0.036000 0.208000 0.100000 0.827000 \n", "75% 0.145250 0.261000 0.187250 0.903000 \n", "max 0.945000 1.000000 1.000000 0.960000 " ] }, "execution_count": 4, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# the built in describe() function calculates several descriptive statististics for each column\n", "# in the data frame and returns them as a new dataframe\n", "df.describe()" ] }, { "cell_type": "code", "execution_count": 5, "id": "ee9e8328", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "count 2048.000000\n", "mean 0.237641\n", "std 0.160780\n", "min 0.000000\n", "25% 0.163000\n", "50% 0.208000\n", "75% 0.261000\n", "max 1.000000\n", "Name: swd_pct, dtype: float64" ] }, "execution_count": 5, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# we can also call describe aon a single series:\n", "df.swd_pct.describe()" ] }, { "cell_type": "code", "execution_count": 6, "id": "4cdd0f0d", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/tmp/ipykernel_4218/792112253.py:8: FutureWarning: The default value of numeric_only in DataFrame.corr is deprecated. In a future version, it will default to False. Select only valid columns or specify the value of numeric_only to silence this warning.\n", " data.corr()\n" ] }, { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
total_enrollmentasian_pctblack_pcthispanic_pctwhite_pctswd_pctell_pctpoverty_pct
total_enrollment1.0000000.350881-0.242553-0.1115510.179056-0.175962-0.018582-0.155614
asian_pct0.3508811.000000-0.452366-0.3702640.210809-0.2104890.138261-0.287675
black_pct-0.242553-0.4523661.000000-0.416505-0.4508010.138008-0.3627940.299685
hispanic_pct-0.111551-0.370264-0.4165051.000000-0.3967390.0704210.4418780.490632
white_pct0.1790560.210809-0.450801-0.3967391.000000-0.083534-0.177996-0.795595
swd_pct-0.175962-0.2104890.1380080.070421-0.0835341.0000000.0001170.233832
ell_pct-0.0185820.138261-0.3627940.441878-0.1779960.0001171.0000000.359285
poverty_pct-0.155614-0.2876750.2996850.490632-0.7955950.2338320.3592851.000000
\n", "
" ], "text/plain": [ " total_enrollment asian_pct black_pct hispanic_pct \\\n", "total_enrollment 1.000000 0.350881 -0.242553 -0.111551 \n", "asian_pct 0.350881 1.000000 -0.452366 -0.370264 \n", "black_pct -0.242553 -0.452366 1.000000 -0.416505 \n", "hispanic_pct -0.111551 -0.370264 -0.416505 1.000000 \n", "white_pct 0.179056 0.210809 -0.450801 -0.396739 \n", "swd_pct -0.175962 -0.210489 0.138008 0.070421 \n", "ell_pct -0.018582 0.138261 -0.362794 0.441878 \n", "poverty_pct -0.155614 -0.287675 0.299685 0.490632 \n", "\n", " white_pct swd_pct ell_pct poverty_pct \n", "total_enrollment 0.179056 -0.175962 -0.018582 -0.155614 \n", "asian_pct 0.210809 -0.210489 0.138261 -0.287675 \n", "black_pct -0.450801 0.138008 -0.362794 0.299685 \n", "hispanic_pct -0.396739 0.070421 0.441878 0.490632 \n", "white_pct 1.000000 -0.083534 -0.177996 -0.795595 \n", "swd_pct -0.083534 1.000000 0.000117 0.233832 \n", "ell_pct -0.177996 0.000117 1.000000 0.359285 \n", "poverty_pct -0.795595 0.233832 0.359285 1.000000 " ] }, "execution_count": 6, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# we can also call the corr() method to show correclations between columns\n", "# we will take out \"district\" from this data because the district number\n", "# is categorical -- not the measure of a value\n", "\n", "# correlations close to 1 or negative one show high correlations\n", "# closer to zero items are not closely correlated\n", "data = df.drop(columns=[\"district\"])\n", "data.corr()" ] }, { "cell_type": "code", "execution_count": 7, "id": "2b796861", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/tmp/ipykernel_4218/143913075.py:3: FutureWarning: The default value of numeric_only in DataFrame.corr is deprecated. In a future version, it will default to False. Select only valid columns or specify the value of numeric_only to silence this warning.\n", " corr = data.corr()\n" ] }, { "data": { "text/html": [ "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
 total_enrollmentasian_pctblack_pcthispanic_pctwhite_pctswd_pctell_pctpoverty_pct
total_enrollment1.0000000.350881-0.242553-0.1115510.179056-0.175962-0.018582-0.155614
asian_pct0.3508811.000000-0.452366-0.3702640.210809-0.2104890.138261-0.287675
black_pct-0.242553-0.4523661.000000-0.416505-0.4508010.138008-0.3627940.299685
hispanic_pct-0.111551-0.370264-0.4165051.000000-0.3967390.0704210.4418780.490632
white_pct0.1790560.210809-0.450801-0.3967391.000000-0.083534-0.177996-0.795595
swd_pct-0.175962-0.2104890.1380080.070421-0.0835341.0000000.0001170.233832
ell_pct-0.0185820.138261-0.3627940.441878-0.1779960.0001171.0000000.359285
poverty_pct-0.155614-0.2876750.2996850.490632-0.7955950.2338320.3592851.000000
\n" ], "text/plain": [ "" ] }, "execution_count": 7, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# last, we can use styles to make the correlation table easier to read\n", "# note: you need to run this cell to see the colors -- it's get saved without the styled output\n", "corr = data.corr()\n", "# a coolwarm color map will show values in a gradient where -1 is the deepest blue and 1 is deepest red\n", "corr.style.background_gradient(cmap='coolwarm')" ] } ], "metadata": { "kernelspec": { "display_name": "school-data", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.6" }, "vscode": { "interpreter": { "hash": "c853444e20c489e5b96d8e1a4533affead1d94f1ba40ff9ef08cffb9c8ee794e" } } }, "nbformat": 4, "nbformat_minor": 5 }