diff --git a/notebooks/official/datasets/get_started_bq_datasets.ipynb b/notebooks/official/datasets/get_started_bq_datasets.ipynb index 2e87f46cf..d8a3968c5 100644 --- a/notebooks/official/datasets/get_started_bq_datasets.ipynb +++ b/notebooks/official/datasets/get_started_bq_datasets.ipynb @@ -180,27 +180,14 @@ }, "outputs": [], "source": [ - "import os\n", - "\n", - "# The Vertex AI Workbench Notebook product has specific requirements\n", - "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\") and not os.getenv(\"VIRTUAL_ENV\")\n", - "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n", - " \"/opt/deeplearning/metadata/env_version\"\n", - ")\n", - "\n", - "# Vertex AI Notebook requires dependencies to be installed with '--user'\n", - "USER_FLAG = \"\"\n", - "if IS_WORKBENCH_NOTEBOOK:\n", - " USER_FLAG = \"--user\"\n", - "\n", - "! pip3 install --upgrade --quiet {USER_FLAG} google-cloud-aiplatform \\\n", - " google-cloud-bigquery \\\n", - " tensorflow \\\n", - " tensorflow-io==0.18 \\\n", - " xgboost \\\n", - " numpy \\\n", - " pandas \\\n", - " pyarrow" + "! pip3 install --upgrade --quiet google-cloud-aiplatform \\\n", + " google-cloud-bigquery \\\n", + " tensorflow \\\n", + " tensorflow-io==0.18 \\\n", + " xgboost \\\n", + " numpy \\\n", + " pandas \\\n", + " pyarrow" ] }, { @@ -209,108 +196,52 @@ "id": "restart" }, "source": [ - "### Restart the kernel\n", - "\n", - "Once you've installed the additional packages, you need to restart the notebook kernel so it can find the packages." + "### Colab only: Uncomment the following cell to restart the kernel" ] }, { "cell_type": "code", "execution_count": null, "metadata": { - "id": "restart" + "id": "D-ZBOjErv5mM" }, "outputs": [], "source": [ - "import sys\n", + "# Automatically restart kernel after installs so that your environment can access the new packages\n", + "# import IPython\n", "\n", - "if \"google.colab\" in sys.modules:\n", - " # Automatically restart kernel after installs\n", - " import IPython\n", - "\n", - " app = IPython.Application.instance()\n", - " app.kernel.do_shutdown(True)" + "# app = IPython.Application.instance()\n", + "# app.kernel.do_shutdown(True)" ] }, { "cell_type": "markdown", "metadata": { - "id": "BF1j6f9HApxa" + "id": "yfEglUHQk9S3" }, "source": [ - "### Set up your Google Cloud project\n", + "## Before you begin\n", "\n", - "**The following steps are required, regardless of your notebook environment.**\n", + "### Set your project ID\n", "\n", - "1. Select or create a Google Cloud project. When you first create an account, you get a $300 free credit towards your compute/storage costs.\n", - "\n", - "1. Make sure that billing is enabled for your project.\n", - "\n", - "1. Enable the Vertex AI API.\n", - "\n", - "1. If you are running this notebook locally, you will need to install the Cloud SDK.\n", - "\n", - "1. Enter your project ID in the cell below. Then run the cell to make sure the\n", - "Cloud SDK uses the right project for all the commands in this notebook.\n", - "\n", - "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "WReHDGG5g0XY" - }, - "source": [ - "#### Set your project ID\n", - "\n", - "**If you don't know your project ID**, you may be able to get your project ID using `gcloud`." + "**If you don't know your project ID**, try the following:\n", + "* Run `gcloud config list`.\n", + "* Run `gcloud projects list`.\n", + "* See the support page: [Locate the project ID](https://support.google.com/googleapi/answer/7014113)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { - "id": "3c8049930470" + "id": "set_project_id" }, "outputs": [], "source": [ - "PROJECT_ID = \"[your-project-id]\" # @param {type:\"string\"}" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "oM1iC_MfAts1" - }, - "outputs": [], - "source": [ - "if PROJECT_ID == \"\" or PROJECT_ID is None or PROJECT_ID == \"[your-project-id]\":\n", - " # Get your GCP project id from gcloud\n", - " shell_output = ! gcloud config list --format 'value(core.project)' 2>/dev/null\n", - " PROJECT_ID = shell_output[0]\n", - " print(\"Project ID:\", PROJECT_ID)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "qJYoRfYng0XZ" - }, - "source": [ - "Otherwise, set your project ID here." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "riG_qUokg0XZ" - }, - "outputs": [], - "source": [ - "! gcloud config set project $PROJECT_ID" + "PROJECT_ID = \"[your-project-id]\" # @param {type:\"string\"}\n", + "\n", + "# Set the project id\n", + "! gcloud config set project {PROJECT_ID}" ] }, { @@ -321,16 +252,7 @@ "source": [ "#### Region\n", "\n", - "You can also change the `REGION` variable, which is used for operations\n", - "throughout the rest of this notebook. Below are regions supported for Vertex AI. We recommend that you choose the region closest to you.\n", - "\n", - "- Americas: `us-central1`\n", - "- Europe: `europe-west4`\n", - "- Asia Pacific: `asia-east1`\n", - "\n", - "You might not be able to use a multi-regional bucket for training with Vertex AI. Not all regions provide support for all Vertex AI services.\n", - "\n", - "Learn more about Vertex AI regions." + "You can also change the `REGION` variable used by Vertex AI. Learn more about [Vertex AI regions](https://cloud.google.com/vertex-ai/docs/general/locations)." ] }, { @@ -341,156 +263,93 @@ }, "outputs": [], "source": [ - "REGION = \"[your-region]\" # @param {type: \"string\"}\n", - "\n", - "if REGION == \"[your-region]\":\n", - " REGION = \"us-central1\"" + "REGION = \"us-central1\" # @param {type: \"string\"}" ] }, { "cell_type": "markdown", "metadata": { - "id": "06571eb4063b" - }, - "source": [ - "#### UUID\n", - "\n", - "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a uuid for each instance session, and append it onto the name of resources you create in this tutorial." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "697568e92bd6" - }, - "outputs": [], - "source": [ - "import random\n", - "import string\n", - "\n", - "\n", - "# Generate a uuid of a specifed length(default=8)\n", - "def generate_uuid(length: int = 8) -> str:\n", - " return \"\".join(random.choices(string.ascii_lowercase + string.digits, k=length))\n", - "\n", - "\n", - "UUID = generate_uuid()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "dr--iN2kAylZ" + "id": "gcp_authenticate" }, "source": [ "### Authenticate your Google Cloud account\n", "\n", - "**If you are using Vertex AI Workbench Notebooks**, your environment is already\n", - "authenticated. \n", + "Depending on your Jupyter environment, you may have to manually authenticate. Follow the relevant instructions below.\n", "\n", - "**If you are using Colab**, run the cell below and follow the instructions\n", - "when prompted to authenticate your account via oAuth.\n", + "**1. Vertex AI Workbench**\n", + "* Do nothing as you are already authenticated.\n", "\n", - "**Otherwise**, follow these steps:\n", - "\n", - "1. In the Cloud Console, go to the **Create service account key** page.\n", - "\n", - "2. Click **Create service account**.\n", - "\n", - "3. In the **Service account name** field, enter a name, and\n", - " click **Create**.\n", - "\n", - "4. In the **Grant this service account access to project** section, click the **Role** drop-down list. Type \"Vertex AI\"\n", - "into the filter box, and select\n", - " **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n", - "\n", - "5. Click *Create*. A JSON file that contains your key downloads to your\n", - "local environment.\n", - "\n", - "6. Enter the path to your service account key as the\n", - "`GOOGLE_APPLICATION_CREDENTIALS` variable in the cell below and run the cell." + "**2. Local JupyterLab instance, uncomment and run:**" ] }, { "cell_type": "code", "execution_count": null, "metadata": { - "id": "PyQmSRbKA8r-" + "id": "ce6043da7b33" }, "outputs": [], "source": [ - "# If you are running this notebook in Colab, run this cell and follow the\n", - "# instructions to authenticate your GCP account. This provides access to your\n", - "# Cloud Storage bucket and lets you submit training jobs and prediction\n", - "# requests.\n", - "\n", - "import os\n", - "import sys\n", - "\n", - "# If on Vertex AI Workbench, then don't execute this code\n", - "IS_COLAB = \"google.colab\" in sys.modules\n", - "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n", - " \"DL_ANACONDA_HOME\"\n", - "):\n", - " if \"google.colab\" in sys.modules:\n", - " from google.colab import auth as google_auth\n", - "\n", - " google_auth.authenticate_user()\n", - "\n", - " # If you are running this notebook locally, replace the string below with the\n", - " # path to your service account key and run this cell to authenticate your GCP\n", - " # account.\n", - " elif not os.getenv(\"IS_TESTING\"):\n", - " %env GOOGLE_APPLICATION_CREDENTIALS ''" + "# ! gcloud auth login" ] }, { "cell_type": "markdown", "metadata": { - "id": "62f861b68b50" + "id": "0367eac06a10" + }, + "source": [ + "**3. Colab, uncomment and run:**" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "21ad4dbb4a61" + }, + "outputs": [], + "source": [ + "# from google.colab import auth\n", + "# auth.authenticate_user()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "c13224697bfb" + }, + "source": [ + "**4. Service account or other**\n", + "* See how to grant Cloud Storage permissions to your service account at https://cloud.google.com/storage/docs/gsutil/commands/iam#ch-examples." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "bucket:mbsdk" }, "source": [ "### Create a Cloud Storage bucket\n", "\n", - "**The following steps are required, regardless of your notebook environment.**\n", - "\n", - "To update your model artifacts without re-building the container, you upload your model\n", - "artifacts and any custom code to a Cloud Storage bucket. You also provide a Cloud Storage bucket to serve as a default staging location for your Vertex AI SDK.\n", - "\n", - "Set the name of your Cloud Storage bucket below. It must be unique across all\n", - "Cloud Storage buckets. " + "Create a storage bucket to store intermediate artifacts such as datasets." ] }, { "cell_type": "code", "execution_count": null, "metadata": { - "id": "2232344edb11" + "id": "bucket" }, "outputs": [], "source": [ - "BUCKET_NAME = \"[your-bucket-name]\" # @param {type:\"string\"}\n", - "BUCKET_URI = f\"gs://{BUCKET_NAME}\"" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "6749c55b3f6f" - }, - "outputs": [], - "source": [ - "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"[your-bucket-name]\":\n", - " BUCKET_NAME = PROJECT_ID + \"aip-\" + UUID\n", - " BUCKET_URI = f\"gs://{BUCKET_NAME}\"" + "BUCKET_URI = f\"gs://your-bucket-name-{PROJECT_ID}-unique\" # @param {type:\"string\"}" ] }, { "cell_type": "markdown", "metadata": { - "id": "58cb4f5895f0" + "id": "create_bucket" }, "source": [ "**Only if your bucket doesn't already exist**: Run the following cell to create your Cloud Storage bucket." @@ -500,33 +359,13 @@ "cell_type": "code", "execution_count": null, "metadata": { - "id": "2d2208676cee" + "id": "create_bucket" }, "outputs": [], "source": [ "! gsutil mb -l $REGION $BUCKET_URI" ] }, - { - "cell_type": "markdown", - "metadata": { - "id": "c664a5abc11a" - }, - "source": [ - "Finally, validate access to your Cloud Storage bucket by examining its contents:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "2c1b1c29f5f6" - }, - "outputs": [], - "source": [ - "! gsutil ls -al $BUCKET_URI" - ] - }, { "cell_type": "markdown", "metadata": { @@ -645,7 +484,7 @@ "outputs": [], "source": [ "dataset = aiplatform.TabularDataset.create(\n", - " display_name=\"NOAA historical weather data\" + \"_\" + UUID,\n", + " display_name=\"NOAA historical weather data\",\n", " bq_source=[IMPORT_FILE],\n", " labels={\"user_metadata\": BUCKET_URI[5:]},\n", ")\n", @@ -720,7 +559,7 @@ "gcs_source = IMPORT_FILES\n", "\n", "dataset = aiplatform.TabularDataset.create(\n", - " display_name=\"NOAA historical weather data\" + \"_\" + UUID,\n", + " display_name=\"NOAA historical weather data\",\n", " gcs_source=gcs_source,\n", " labels={\"user_metadata\": BUCKET_URI[5:]},\n", ")\n", @@ -762,10 +601,10 @@ " or BQ_MY_DATASET is None\n", " or BQ_MY_DATASET == \"[your-dataset-name]\"\n", "):\n", - " BQ_MY_DATASET = \"mlops_dataset_\" + UUID\n", + " BQ_MY_DATASET = \"mlops_dataset\"\n", "\n", "if BQ_MY_TABLE == \"\" or BQ_MY_TABLE is None or BQ_MY_TABLE == \"[your-view-name]\":\n", - " BQ_MY_TABLE = \"mlops_view_\" + UUID" + " BQ_MY_TABLE = \"mlops_view\"" ] }, {