Updating dataset and other changes from review

notebooks/community/exploratory_data_analysis/eda_with_r_and_bigquery.ipynb

@@ -31,9 +31,9 @@
    "source": [
     "# Exploratory Data Analysis with R and BigQuery\n",
     "\n",
-    "**Authors**:  [Alok Pattani](https://github.com/alokpattani), [Khalid Salama](https://github.com/ksalama)\n",
+    "**Author**:  [Alok Pattani](https://github.com/alokpattani)\n",
     "\n",
-    "**Last Updated**:  February 2024\n",
+    "**Last Updated**:  July 2024\n",
     "\n",
     "## Overview\n",
     "\n",
@@ -79,7 +79,9 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {},
+   "metadata": {
+    "tags": []
+   },
    "outputs": [],
    "source": [
     "version"
@@ -135,7 +137,9 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {},
+   "metadata": {
+    "tags": []
+   },
    "outputs": [],
    "source": [
     "bq_auth(use_oob = TRUE)"
@@ -151,7 +155,9 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {},
+   "metadata": {
+    "tags": []
+   },
    "outputs": [],
    "source": [
     "# Set the project ID\n",
@@ -168,13 +174,27 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {},
+   "metadata": {
+    "tags": []
+   },
    "outputs": [],
    "source": [
     "# Set your Cloud Storage bucket name\n",
     "BUCKET_NAME <- \"[YOUR-BUCKET-NAME]\""
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "# Set default height/width for plots generated\n",
+    "options(repr.plot.height = 9, repr.plot.width = 16)"
+   ]
+  },
   {
    "cell_type": "markdown",
    "metadata": {},
@@ -192,31 +212,64 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {},
+   "metadata": {
+    "tags": []
+   },
    "outputs": [],
    "source": [
     "sql_query_template <- \"\n",
     "    SELECT\n",
-    "      ROUND(weight_pounds, 2) AS weight_pounds,\n",
-    "      is_male,\n",
-    "      mother_age,\n",
-    "      plurality,\n",
-    "      gestation_weeks,\n",
-    "      cigarette_use,\n",
-    "      alcohol_use,\n",
-    "      CAST(ABS(FARM_FINGERPRINT(CONCAT(\n",
-    "        CAST(YEAR AS STRING), CAST(month AS STRING), \n",
-    "        CAST(weight_pounds AS STRING)))\n",
-    "        ) AS STRING) AS key\n",
+    "      TIMESTAMP_DIFF(dropoff_datetime, pickup_datetime, MINUTE) AS trip_time_minutes,  \n",
+    "\n",
+    "      passenger_count,\n",
+    "\n",
+    "      ROUND(trip_distance, 1) AS trip_distance_miles,\n",
+    "\n",
+    "      rate_code,\n",
+    "      /* Mapping from rate code to type from description column in BQ table schema */\n",
+    "      (CASE \n",
+    "        WHEN rate_code = '1.0'\n",
+    "          THEN 'Standard rate'\n",
+    "        WHEN rate_code = '2.0'\n",
+    "          THEN 'JFK'\n",
+    "        WHEN rate_code = '3.0'\n",
+    "          THEN 'Newark'\n",
+    "        WHEN rate_code = '4.0'\n",
+    "          THEN 'Nassau or Westchester'\n",
+    "        WHEN rate_code = '5.0'\n",
+    "          THEN 'Negotiated fare'\n",
+    "        WHEN rate_code = '6.0'\n",
+    "          THEN 'Group ride'\n",
+    "        /* Several NULL AND some '99.0' values go here */\n",
+    "        ELSE 'Unknown'\n",
+    "        END)\n",
+    "        AS rate_type,\n",
+    "\n",
+    "      fare_amount,\n",
+    "\n",
+    "      CAST(ABS(FARM_FINGERPRINT(\n",
+    "        CONCAT(\n",
+    "          CAST(trip_distance AS STRING), \n",
+    "          CAST(fare_amount AS STRING)\n",
+    "          )\n",
+    "        ))\n",
+    "        AS STRING)\n",
+    "        AS key\n",
+    "\n",
     "    FROM\n",
-    "        publicdata.samples.natality\n",
-    "    WHERE \n",
-    "      year > 2000\n",
-    "      AND weight_pounds > 0\n",
-    "      AND mother_age > 0\n",
-    "      AND plurality > 0\n",
-    "      AND gestation_weeks > 0\n",
-    "      AND month > 0\n",
+    "      `bigquery-public-data.new_york_taxi_trips.tlc_yellow_trips_2022`\n",
+    "\n",
+    "    /* Filter out some outlier or hard to understand values */\n",
+    "    WHERE\n",
+    "      (TIMESTAMP_DIFF(dropoff_datetime, pickup_datetime, MINUTE)\n",
+    "        BETWEEN 0.01 AND 120)\n",
+    "      AND\n",
+    "      (passenger_count BETWEEN 1 AND 10)\n",
+    "      AND\n",
+    "      (trip_distance BETWEEN 0.01 AND 100)\n",
+    "      AND\n",
+    "      (fare_amount BETWEEN 0.01 AND 250)\n",
+    "\n",
     "    LIMIT %s\n",
     "\""
    ]
@@ -232,14 +285,16 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {},
+   "metadata": {
+    "tags": []
+   },
    "outputs": [],
    "source": [
     "sample_size <- 10000\n",
     "\n",
     "sql_query <- sprintf(sql_query_template, sample_size)\n",
     "\n",
-    "natality_data <- bq_table_download(\n",
+    "taxi_trip_data <- bq_table_download(\n",
     "    bq_project_query(\n",
     "        PROJECT_ID, \n",
     "        query = sql_query\n",
@@ -257,31 +312,37 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {},
+   "metadata": {
+    "tags": []
+   },
    "outputs": [],
    "source": [
     "# View the query result\n",
-    "head(natality_data)"
+    "head(taxi_trip_data)"
    ]
   },
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {},
+   "metadata": {
+    "tags": []
+   },
    "outputs": [],
    "source": [
     "# Show # of rows and data types of each column\n",
-    "str(natality_data)"
+    "str(taxi_trip_data)"
    ]
   },
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {},
+   "metadata": {
+    "tags": []
+   },
    "outputs": [],
    "source": [
     "# View the results summary\n",
-    "summary(natality_data)"
+    "summary(taxi_trip_data)"
    ]
   },
   {
@@ -294,27 +355,31 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {},
+   "metadata": {
+    "tags": []
+   },
    "outputs": [],
    "source": [
-    "# Display the distribution of baby weights using a histogram\n",
+    "# Display the distribution of fare amounts using a histogram\n",
     "ggplot(\n",
-    "    data = natality_data, \n",
-    "    aes(x = weight_pounds)\n",
+    "    data = taxi_trip_data, \n",
+    "    aes(x = fare_amount)\n",
     "    ) + \n",
-    "geom_histogram(bins = 200)"
+    "geom_histogram(bins = 100)"
    ]
   },
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {},
+   "metadata": {
+    "tags": []
+   },
    "outputs": [],
    "source": [
-    "# Display the relationship between gestation weeks and baby weights \n",
+    "# Display the relationship between trip distance and fare amount\n",
     "ggplot(\n",
-    "    data = natality_data, \n",
-    "    aes(x = gestation_weeks, y = weight_pounds)\n",
+    "    data = taxi_trip_data, \n",
+    "    aes(x = trip_distance_miles, y = fare_amount)\n",
     "    ) + \n",
     "geom_point() + \n",
     "geom_smooth(method = \"lm\")"
@@ -325,23 +390,41 @@
    "metadata": {},
    "source": [
     "### Performing the processing in BigQuery\n",
-    "Create a function that finds the number of records and the average weight for each value of the chosen column."
+    "Create a function that finds the number of trips and the average fare amount for each value of the chosen column."
    ]
   },
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {},
+   "metadata": {
+    "tags": []
+   },
    "outputs": [],
    "source": [
-    "get_distinct_values <- function(column_name) {\n",
+    "get_distinct_value_aggregates <- function(column) {\n",
     "    query <- paste0(\n",
-    "        'SELECT ', column_name, ', \n",
-    "            COUNT(1) AS num_babies,\n",
-    "            AVG(weight_pounds) AS avg_wt\n",
-    "        FROM publicdata.samples.natality\n",
-    "        WHERE year > 2000\n",
-    "        GROUP BY ', column_name)\n",
+    "        'SELECT ', \n",
+    "          column, \n",
+    "          ', \n",
+    "          COUNT(1) AS num_trips,\n",
+    "          AVG(fare_amount) AS avg_fare_amount\n",
+    "        \n",
+    "        FROM\n",
+    "          `bigquery-public-data.new_york_taxi_trips.tlc_yellow_trips_2022`\n",
+    "        \n",
+    "        WHERE\n",
+    "          (TIMESTAMP_DIFF(dropoff_datetime, pickup_datetime, MINUTE) \n",
+    "            BETWEEN 0.01 AND 120)\n",
+    "          AND\n",
+    "          (passenger_count BETWEEN 1 AND 10)\n",
+    "          AND\n",
+    "          (trip_distance BETWEEN 0.01 AND 100)\n",
+    "          AND\n",
+    "          (fare_amount BETWEEN 0.01 AND 250)\n",
+    "        \n",
+    "        GROUP BY 1\n",
+    "        '\n",
+    "        )\n",
     "    \n",
     "    bq_table_download(\n",
     "        bq_project_query(\n",
@@ -362,20 +445,23 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {},
+   "metadata": {
+    "tags": []
+   },
    "outputs": [],
    "source": [
-    "df <- get_distinct_values('mother_age')\n",
+    "df <- get_distinct_value_aggregates(\n",
+    "  'TIMESTAMP_DIFF(dropoff_datetime, pickup_datetime, MINUTE) AS trip_time_minutes')\n",
     "\n",
     "ggplot(\n",
     "    data = df, \n",
-    "    aes(x = mother_age, y = num_babies)\n",
+    "    aes(x = trip_time_minutes, y = num_trips)\n",
     "    ) + \n",
     "geom_line()\n",
     "\n",
     "ggplot(\n",
-    "    data = df, \n",
-    "    aes(x = mother_age, y = avg_wt)\n",
+    "    data = df,\n",
+    "    aes(x = trip_time_minutes, y = avg_fare_amount)\n",
     "    ) + \n",
     "geom_line()"
    ]
@@ -383,64 +469,88 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {},
+   "metadata": {
+    "tags": []
+   },
    "outputs": [],
    "source": [
-    "df <- get_distinct_values('is_male')\n",
+    "df <- get_distinct_value_aggregates('passenger_count')\n",
     "\n",
     "ggplot(\n",
     "    data = df, \n",
-    "    aes(x = is_male, y = num_babies)\n",
+    "    aes(x = passenger_count, y = num_trips)\n",
+    "    ) + \n",
+    "geom_col() +\n",
+    "scale_x_continuous(breaks = 1:10)\n",
+    "\n",
+    "ggplot(\n",
+    "    data = df, \n",
+    "    aes(x = passenger_count, y = avg_fare_amount)\n",
+    "    ) + \n",
+    "geom_col() +\n",
+    "scale_x_continuous(breaks = 1:10)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "df <- get_distinct_value_aggregates('ROUND(trip_distance, 0) AS trip_distance_miles')\n",
+    "\n",
+    "ggplot(\n",
+    "    data = df, \n",
+    "    aes(x = trip_distance_miles, y = num_trips)\n",
+    "    ) + \n",
+    "geom_line()\n",
+    "\n",
+    "ggplot(\n",
+    "    data = df,\n",
+    "    aes(x = trip_distance_miles, y = avg_fare_amount)\n",
+    "    ) + \n",
+    "geom_line()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "df <- get_distinct_value_aggregates(\"\n",
+    "  (CASE \n",
+    "    WHEN rate_code = '1.0'\n",
+    "      THEN 'Standard rate'\n",
+    "    WHEN rate_code = '2.0'\n",
+    "      THEN 'JFK'\n",
+    "    WHEN rate_code = '3.0'\n",
+    "      THEN 'Newark'\n",
+    "    WHEN rate_code = '4.0'\n",
+    "      THEN 'Nassau or Westchester'\n",
+    "    WHEN rate_code = '5.0'\n",
+    "      THEN 'Negotiated fare'\n",
+    "    WHEN rate_code = '6.0'\n",
+    "      THEN 'Group ride'\n",
+    "    /* Several NULL AND some '99.0' values go here */\n",
+    "    ELSE 'Unknown'\n",
+    "    END)\n",
+    "    AS rate_type\n",
+    "  \")\n",
+    "\n",
+    "ggplot(\n",
+    "    data = df,\n",
+    "    aes(x = rate_type, y = num_trips)\n",
     "    ) + \n",
     "geom_col()\n",
     "\n",
     "ggplot(\n",
-    "    data = df, \n",
-    "    aes(x = is_male, y = avg_wt)\n",
-    "    ) + \n",
-    "geom_col()"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "df <- get_distinct_values('plurality')\n",
-    "\n",
-    "ggplot(\n",
-    "    data = df, \n",
-    "    aes(x = plurality, y = num_babies)\n",
-    "    ) + \n",
-    "geom_col() + \n",
-    "scale_y_log10()\n",
-    "\n",
-    "ggplot(\n",
     "    data = df,\n",
-    "    aes(x = plurality, y = avg_wt)\n",
-    "    ) + \n",
-    "geom_col()"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "df <- get_distinct_values('gestation_weeks')\n",
-    "\n",
-    "ggplot(\n",
-    "    data = df,\n",
-    "    aes(x = gestation_weeks, y = num_babies)\n",
-    "    ) + \n",
-    "geom_col() + \n",
-    "scale_y_log10()\n",
-    "\n",
-    "ggplot(\n",
-    "    data = df,\n",
-    "    aes(x = gestation_weeks, y = avg_wt)\n",
+    "    aes(x = rate_type, y = avg_fare_amount)\n",
     "    ) + \n",
     "geom_col()"
    ]
@@ -455,7 +565,9 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {},
+   "metadata": {
+    "tags": []
+   },
    "outputs": [],
    "source": [
     "# Prepare training and evaluation data from BigQuery\n",
@@ -489,7 +601,9 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {},
+   "metadata": {
+    "tags": []
+   },
    "outputs": [],
    "source": [
     "print(paste0(\"Training instances count: \", nrow(train_data)))\n",
@@ -500,23 +614,27 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {},
+   "metadata": {
+    "tags": []
+   },
    "outputs": [],
    "source": [
-    "# Write data frames to local CSV files, without headers or row names\n",
+    "# Write data frames to local CSV files, with headers\n",
     "dir.create(file.path('data'), showWarnings = FALSE)\n",
     "\n",
     "write.table(train_data, \"data/train_data.csv\", \n",
-    "   row.names = FALSE, col.names = FALSE, sep = \",\")\n",
+    "   row.names = FALSE, col.names = TRUE, sep = \",\")\n",
     "\n",
     "write.table(eval_data, \"data/eval_data.csv\", \n",
-    "   row.names = FALSE, col.names = FALSE, sep = \",\")"
+    "   row.names = FALSE, col.names = TRUE, sep = \",\")"
    ]
   },
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {},
+   "metadata": {
+    "tags": []
+   },
    "outputs": [],
    "source": [
     "# Upload CSV data to Cloud Storage by passing gsutil commands to system\n",
@@ -541,9 +659,9 @@
  "metadata": {
   "environment": {
    "kernel": "conda-env-r-r",
-   "name": "workbench-notebooks.m115",
+   "name": "workbench-notebooks.m123",
    "type": "gcloud",
-   "uri": "gcr.io/deeplearning-platform-release/workbench-notebooks:m115"
+   "uri": "us-docker.pkg.dev/deeplearning-platform-release/gcr.io/workbench-notebooks:m123"
   },
   "kernelspec": {
    "display_name": "R (Local)",