diff --git a/.cloud-build/execute_changed_notebooks_cli.py b/.cloud-build/execute_changed_notebooks_cli.py index 31782fb39..7323d36a2 100755 --- a/.cloud-build/execute_changed_notebooks_cli.py +++ b/.cloud-build/execute_changed_notebooks_cli.py @@ -68,6 +68,12 @@ parser.add_argument( help="A service account. This is used to inject a variable value into the notebook before running. This is not the account that will run the notebook.", required=True, ) +parser.add_argument( + "--variable_vpc_network", + type=str, + help="The full VPC network name. See https://cloud.google.com/compute/docs/networks-and-firewalls#networks. Format is projects/{project}/global/networks/{network}, where {project} is a project number, as in '12345', and {network} is network name. See for details. This is used to inject a variable value into the notebook before running.", + required=False, +) parser.add_argument( "--staging_bucket", type=str, @@ -114,10 +120,11 @@ execute_changed_notebooks_helper.process_and_execute_notebooks( container_uri=args.container_uri, staging_bucket=args.staging_bucket, artifacts_bucket=args.artifacts_bucket, + should_parallelize=args.should_parallelize, + timeout=args.timeout, variable_project_id=args.variable_project_id, variable_region=args.variable_region, variable_service_account=args.variable_service_account, + variable_vpc_network=args.variable_vpc_network, private_pool_id=args.private_pool_id, - should_parallelize=args.should_parallelize, - timeout=args.timeout, ) diff --git a/.cloud-build/execute_changed_notebooks_helper.py b/.cloud-build/execute_changed_notebooks_helper.py index 9b56a4e90..e5830a15c 100755 --- a/.cloud-build/execute_changed_notebooks_helper.py +++ b/.cloud-build/execute_changed_notebooks_helper.py @@ -67,7 +67,7 @@ class NotebookExecutionResult: output_uri: str build_id: str error_message: Optional[str] - + @property def output_uri_web(self) -> Optional[str]: if self.output_uri.startswith("gs://"): @@ -81,6 +81,7 @@ def _process_notebook( variable_project_id: str, variable_region: str, variable_service_account: str, + variable_vpc_network: Optional[str], ): # Read notebook with open(notebook_path) as f: @@ -93,6 +94,7 @@ def _process_notebook( "PROJECT_ID": variable_project_id, "REGION": variable_region, "SERVICE_ACCOUNT": variable_service_account, + "VPC_NETWORK": variable_vpc_network, }, ) @@ -128,8 +130,9 @@ def process_and_execute_notebook( variable_project_id: str, variable_region: str, variable_service_account: str, + variable_vpc_network: Optional[str], private_pool_id: Optional[str], - deadline: datetime, + deadline: datetime.datetime, notebook: str, should_get_tail_logs: bool = False, ) -> NotebookExecutionResult: @@ -137,6 +140,13 @@ def process_and_execute_notebook( print(f"Running notebook: {notebook}") + # Handle empty strings + if not variable_vpc_network: + variable_vpc_network = None + + if not private_pool_id: + private_pool_id = None + # Create paths notebook_output_uri = "/".join([artifacts_bucket, pathlib.Path(notebook).name]) @@ -163,6 +173,7 @@ def process_and_execute_notebook( variable_project_id=variable_project_id, variable_region=variable_region, variable_service_account=variable_service_account, + variable_vpc_network=variable_vpc_network, ) # Upload the pre-processed code to a GCS bucket @@ -266,8 +277,8 @@ def get_changed_notebooks( notebooks = [] else: print(f"Looking for all notebooks.") - notebooks = subprocess.check_output(["git", "ls-files"] + test_paths) - notebooks = notebooks.decode("utf-8").split("\n") + notebooks_str = subprocess.check_output(["git", "ls-files"] + test_paths) + notebooks = notebooks_str.decode("utf-8").split("\n") notebooks = [notebook for notebook in notebooks if notebook.endswith(".ipynb")] notebooks = [notebook for notebook in notebooks if len(notebook) > 0] @@ -286,12 +297,13 @@ def process_and_execute_notebooks( container_uri: str, staging_bucket: str, artifacts_bucket: str, + should_parallelize: bool, + timeout: int, variable_project_id: str, variable_region: str, variable_service_account: str, - private_pool_id: Optional[str], - should_parallelize: bool, - timeout: int, + variable_vpc_network: Optional[str] = None, + private_pool_id: Optional[str] = None, ): """ Run the notebooks that exist under the folders defined in the test_paths_file. @@ -349,6 +361,7 @@ def process_and_execute_notebooks( variable_project_id, variable_region, variable_service_account, + variable_vpc_network, private_pool_id, deadline, ), @@ -364,6 +377,7 @@ def process_and_execute_notebooks( variable_project_id=variable_project_id, variable_region=variable_region, variable_service_account=variable_service_account, + variable_vpc_network=variable_vpc_network, private_pool_id=private_pool_id, deadline=deadline, notebook=notebook, @@ -389,11 +403,18 @@ def process_and_execute_notebooks( format_timedelta(result.duration), result.log_url, result.output_uri, - result.output_uri_web + result.output_uri_web, ] for result in results_sorted ], - headers=["build_tag", "status", "duration", "log_url", "output_uri", "output_uri_web"], + headers=[ + "build_tag", + "status", + "duration", + "log_url", + "output_uri", + "output_uri_web", + ], ) ) @@ -422,6 +443,7 @@ def process_and_execute_notebooks( variable_project_id=variable_project_id, variable_region=variable_region, variable_service_account=variable_service_account, + variable_vpc_network=variable_vpc_network, ) execute_notebook_helper.execute_notebook( diff --git a/.cloud-build/notebook-execution-test-cloudbuild.yaml b/.cloud-build/notebook-execution-test-cloudbuild.yaml index 5019001d2..cf8c9a1d6 100644 --- a/.cloud-build/notebook-execution-test-cloudbuild.yaml +++ b/.cloud-build/notebook-execution-test-cloudbuild.yaml @@ -36,7 +36,7 @@ steps: - -c - | . workspace/env/bin/activate && - python3 .cloud-build/execute_changed_notebooks_cli.py --test_paths_file "${_TEST_PATHS_FILE}" --base_branch "${_FORCED_BASE_BRANCH}" --container_uri ${_PYTHON_IMAGE} --staging_bucket ${_GCS_STAGING_BUCKET} --artifacts_bucket ${_GCS_STAGING_BUCKET}/executed_notebooks/PR_${_PR_NUMBER}/BUILD_${BUILD_ID} --variable_project_id ${PROJECT_ID} --variable_region ${_GCP_REGION} --variable_service_account ${_GCP_SERVICE_ACCOUNT} `if [ ! -z "${_PRIVATE_POOL_NAME}" ]; then echo "--private_pool_id ${_PRIVATE_POOL_NAME}"; fi` + python3 .cloud-build/execute_changed_notebooks_cli.py --test_paths_file "${_TEST_PATHS_FILE}" --base_branch "${_FORCED_BASE_BRANCH}" --container_uri ${_PYTHON_IMAGE} --staging_bucket ${_GCS_STAGING_BUCKET} --artifacts_bucket ${_GCS_STAGING_BUCKET}/executed_notebooks/PR_${_PR_NUMBER}/BUILD_${BUILD_ID} --variable_project_id ${PROJECT_ID} --variable_region ${_GCP_REGION} --variable_service_account ${_GCP_SERVICE_ACCOUNT} --variable_vpc_network "${_GPC_VPC_NETWORK_NAME}" `if [ ! -z "${_PRIVATE_POOL_NAME}" ]; then echo "--private_pool_id ${_PRIVATE_POOL_NAME}"; fi` env: - 'IS_TESTING=1' timeout: 86400s diff --git a/.cloud-build/test_notebook_vm.txt b/.cloud-build/test_notebook_vm.txt index 29840423b..92094ddc0 100644 --- a/.cloud-build/test_notebook_vm.txt +++ b/.cloud-build/test_notebook_vm.txt @@ -2,3 +2,4 @@ notebooks/official/vizier/gapic-vizier-multi-objective-optimization.ipynb notebooks/official/pipelines/lightweight_functions_component_io_kfp.ipynb notebooks/official/ml_metadata/sdk-metric-parameter-tracking-for-locally-trained-models.ipynb notebooks/official/pipelines/metrics_viz_run_compare_kfp.ipynb +notebooks/official/matching_engine/sdk_matching_engine_for_indexing.ipynb \ No newline at end of file diff --git a/notebooks/official/CODEOWNERS b/notebooks/official/CODEOWNERS index e52eeecf9..c08ba789d 100644 --- a/notebooks/official/CODEOWNERS +++ b/notebooks/official/CODEOWNERS @@ -5,7 +5,7 @@ * @GoogleCloudPlatform/vertex-ai-samples-contributors @GoogleCloudPlatform/caiis-tw # matching_engine folder -/matching_engine @shenzhimo2 +/matching_engine @shenzhimo2 @ivanmkc /tabnet/tabnet_vertex_tutorial.ipynb @longtle diff --git a/notebooks/community/matching_engine/sdk_matching_engine_for_indexing.ipynb b/notebooks/official/matching_engine/sdk_matching_engine_for_indexing.ipynb similarity index 88% rename from notebooks/community/matching_engine/sdk_matching_engine_for_indexing.ipynb rename to notebooks/official/matching_engine/sdk_matching_engine_for_indexing.ipynb index 41e4fa68f..1910d2ff5 100644 --- a/notebooks/community/matching_engine/sdk_matching_engine_for_indexing.ipynb +++ b/notebooks/official/matching_engine/sdk_matching_engine_for_indexing.ipynb @@ -32,13 +32,13 @@ "\n", "\n", " \n", "
\n", - " \n", + " \n", " \"Vertex\n", " Run in Vertex Workbench\n", " \n", " \n", - " \n", + " \n", " \"GitHub\n", " View on GitHub\n", " \n", @@ -95,8 +95,182 @@ "id": "S5zc4kbEiYCm" }, "source": [ - "## Before you begin\n", + "## Before you begin" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "d1e95a984673" + }, + "source": [ + "### Set up your Google Cloud project\n", "\n", + "**The following steps are required, regardless of your notebook environment.**\n", + "\n", + "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager).\n", + "\n", + "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n", + "\n", + "1. [Enable the Vertex AI API and Compute Engine API, and Service Networking API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,compute_component,servicenetworking.googleapis.com).\n", + "\n", + "1. Enter your project ID in the cell below. Then run the cell to make sure the\n", + "Cloud SDK uses the right project for all the commands in this notebook.\n", + "\n", + "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "2b9daa35336a" + }, + "source": [ + "### Authenticate your Google Cloud account\n", + "\n", + "**If you are using a Vertex AI Workbench notebook**, your environment is already\n", + "authenticated. Skip this step." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "c6bed8c6a6b3" + }, + "source": [ + "**If you are using Colab**, run the cell below and follow the instructions\n", + "when prompted to authenticate your account via oAuth.\n", + "\n", + "**Otherwise**, follow these steps:\n", + "\n", + "1. In the Cloud Console, go to the [**Create service account key**\n", + " page](https://console.cloud.google.com/apis/credentials/serviceaccountkey).\n", + "\n", + "2. Click **Create service account**.\n", + "\n", + "3. In the **Service account name** field, enter a name, and\n", + " click **Create**.\n", + "\n", + "4. In the **Grant this service account access to project** section, click the **Role** drop-down list. Type \"Vertex AI\"\n", + "into the filter box, and select\n", + " **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n", + "\n", + "5. Click *Create*. A JSON file that contains your key downloads to your\n", + "local environment.\n", + "\n", + "6. Enter the path to your service account key as the\n", + "`GOOGLE_APPLICATION_CREDENTIALS` variable in the cell below and run the cell." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "3e2b43c2d2bf" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Your browser has been opened to visit:\n", + "\n", + " https://accounts.google.com/o/oauth2/auth?response_type=code&client_id=32555940559.apps.googleusercontent.com&redirect_uri=http%3A%2F%2Flocalhost%3A8085%2F&scope=openid+https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fuserinfo.email+https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fcloud-platform+https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fappengine.admin+https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fsqlservice.login+https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fcompute+https%3A%2F%2Fwww.googleapis.com%2Fauth%2Faccounts.reauth&state=UY9jjYfhoSedWWUOWXp5Pmicq0Ic04&access_type=offline&code_challenge=OQefcewSwkT7ZwfzzOVidtngvZspdY1NgN6rltw8x7A&code_challenge_method=S256\n", + "\n" + ] + } + ], + "source": [ + "import os\n", + "import sys\n", + "\n", + "# If you are running this notebook in Colab, run this cell and follow the\n", + "# instructions to authenticate your GCP account. This provides access to your\n", + "# Cloud Storage bucket and lets you submit training jobs and prediction\n", + "# requests.\n", + "\n", + "# The Vertex AI Workbench notebook product has specific requirements\n", + "IS_VERTEX_AI_WORKBENCH_NOTEBOOK = os.path.exists(\n", + " \"/opt/deeplearning/metadata/env_version\"\n", + ")\n", + "\n", + "# If on a Vertex AI Workbench notebook, then don't execute this code\n", + "if not IS_VERTEX_AI_WORKBENCH_NOTEBOOK:\n", + " if \"google.colab\" in sys.modules:\n", + " from google.colab import auth as google_auth\n", + "\n", + " google_auth.authenticate_user()\n", + "\n", + " # If you are running this notebook locally, log in using gcloud\n", + " elif not os.getenv(\"IS_TESTING\"):\n", + " ! gcloud auth login" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "WReHDGG5g0XY" + }, + "source": [ + "#### Set your project ID\n", + "\n", + "**If you don't know your project ID**, you may be able to get your project ID using `gcloud`." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "id": "beb72f394541" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Project ID: python-docs-samples-tests\n" + ] + } + ], + "source": [ + "import os\n", + "\n", + "PROJECT_ID = \"[your-project-id]\" # @param {type:\"string\"}\n", + "\n", + "# Get your Google Cloud project ID from gcloud\n", + "if not os.getenv(\"IS_TESTING\"):\n", + " shell_output = !gcloud config list --format 'value(core.project)' 2>/dev/null\n", + " PROJECT_ID = shell_output[0]\n", + " print(\"Project ID: \", PROJECT_ID)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "f4c6d0a9e66c" + }, + "source": [ + "Otherwise, set your project ID here." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "1dc3fa9ac4f7" + }, + "outputs": [], + "source": [ + "if PROJECT_ID == \"\" or PROJECT_ID is None:\n", + " PROJECT_ID = \"\" # @param {type:\"string\"}" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "4962667eec8e" + }, + "source": [ "* **Prepare a VPC network**. To reduce any network overhead that might lead to unnecessary increase in overhead latency, it is best to call the ANN endpoints from your VPC via a direct [VPC Peering](https://cloud.google.com/vertex-ai/docs/general/vpc-peering) connection. \n", " * The following section describes how to setup a VPC Peering connection if you don't have one. \n", " * This is a one-time initial setup task. You can also reuse existing VPC network and skip this section." @@ -110,9 +284,7 @@ }, "outputs": [], "source": [ - "PROJECT_ID = \"python-docs-samples-tests\" # @param {type:\"string\"}\n", - "\n", - "NETWORK_NAME = \"ann-vpc-network\" # @param {type:\"string\"}\n", + "VPC_NETWORK = \"[your-vpc-network-name]\" # @param {type:\"string\"}\n", "\n", "PEERING_RANGE_NAME = \"ann-haystack-range\"" ] @@ -125,24 +297,28 @@ }, "outputs": [], "source": [ - "# Create a VPC network\n", - "! gcloud compute networks create {NETWORK_NAME} --bgp-routing-mode=regional --subnet-mode=auto --project={PROJECT_ID}\n", + "import os\n", "\n", - "# Add necessary firewall rules\n", - "! gcloud compute firewall-rules create {NETWORK_NAME}-allow-icmp --network {NETWORK_NAME} --priority 65534 --project {PROJECT_ID} --allow icmp\n", + "# Remove the if condition to run the encapsulated code\n", + "if not os.getenv(\"IS_TESTING\"):\n", + " # Create a VPC network\n", + " ! gcloud compute networks create {VPC_NETWORK} --bgp-routing-mode=regional --subnet-mode=auto --project={PROJECT_ID}\n", "\n", - "! gcloud compute firewall-rules create {NETWORK_NAME}-allow-internal --network {NETWORK_NAME} --priority 65534 --project {PROJECT_ID} --allow all --source-ranges 10.128.0.0/9\n", + " # Add necessary firewall rules\n", + " ! gcloud compute firewall-rules create {VPC_NETWORK}-allow-icmp --network {VPC_NETWORK} --priority 65534 --project {PROJECT_ID} --allow icmp\n", "\n", - "! gcloud compute firewall-rules create {NETWORK_NAME}-allow-rdp --network {NETWORK_NAME} --priority 65534 --project {PROJECT_ID} --allow tcp:3389\n", + " ! gcloud compute firewall-rules create {VPC_NETWORK}-allow-internal --network {VPC_NETWORK} --priority 65534 --project {PROJECT_ID} --allow all --source-ranges 10.128.0.0/9\n", "\n", - "! gcloud compute firewall-rules create {NETWORK_NAME}-allow-ssh --network {NETWORK_NAME} --priority 65534 --project {PROJECT_ID} --allow tcp:22\n", + " ! gcloud compute firewall-rules create {VPC_NETWORK}-allow-rdp --network {VPC_NETWORK} --priority 65534 --project {PROJECT_ID} --allow tcp:3389\n", "\n", - "# Reserve IP range\n", - "! gcloud compute addresses create {PEERING_RANGE_NAME} --global --prefix-length=16 --network={NETWORK_NAME} --purpose=VPC_PEERING --project={PROJECT_ID} --description=\"peering range\"\n", + " ! gcloud compute firewall-rules create {VPC_NETWORK}-allow-ssh --network {VPC_NETWORK} --priority 65534 --project {PROJECT_ID} --allow tcp:22\n", "\n", - "# Set up peering with service networking\n", - "# Your account must have the \"Compute Network Admin\" role to run the following.\n", - "! gcloud services vpc-peerings connect --service=servicenetworking.googleapis.com --network={NETWORK_NAME} --ranges={PEERING_RANGE_NAME} --project={PROJECT_ID}" + " # Reserve IP range\n", + " ! gcloud compute addresses create {PEERING_RANGE_NAME} --global --prefix-length=16 --network={VPC_NETWORK} --purpose=VPC_PEERING --project={PROJECT_ID} --description=\"peering range\"\n", + "\n", + " # Set up peering with service networking\n", + " # Your account must have the \"Compute Network Admin\" role to run the following.\n", + " ! gcloud services vpc-peerings connect --service=servicenetworking.googleapis.com --network={VPC_NETWORK} --ranges={PEERING_RANGE_NAME} --project={PROJECT_ID}" ] }, { @@ -230,9 +406,6 @@ }, "outputs": [], "source": [ - "# Automatically restart kernel after installs\n", - "import os\n", - "\n", "if not os.getenv(\"IS_TESTING\"):\n", " # Automatically restart kernel after installs\n", " import IPython\n", @@ -241,88 +414,15 @@ " app.kernel.do_shutdown(True)" ] }, - { - "cell_type": "markdown", - "metadata": { - "id": "BF1j6f9HApxa" - }, - "source": [ - "### Set up your Google Cloud project\n", - "\n", - "**The following steps are required, regardless of your notebook environment.**\n", - "\n", - "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager).\n", - "\n", - "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n", - "\n", - "1. [Enable the Vertex AI API and Compute Engine API, and Service Networking API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,compute_component,servicenetworking.googleapis.com).\n", - "\n", - "1. Enter your project ID in the cell below. Then run the cell to make sure the\n", - "Cloud SDK uses the right project for all the commands in this notebook.\n", - "\n", - "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "WReHDGG5g0XY" - }, - "source": [ - "#### Set your project ID\n", - "\n", - "**If you don't know your project ID**, you may be able to get your project ID using `gcloud`." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "oM1iC_MfAts1" - }, - "outputs": [], - "source": [ - "import os\n", - "\n", - "PROJECT_ID = \"python-docs-samples-tests\"\n", - "\n", - "# Get your Google Cloud project ID from gcloud\n", - "if not os.getenv(\"IS_TESTING\"):\n", - " shell_output = !gcloud config list --format 'value(core.project)' 2>/dev/null\n", - " PROJECT_ID = shell_output[0]\n", - " print(\"Project ID: \", PROJECT_ID)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "qJYoRfYng0XZ" - }, - "source": [ - "Otherwise, set your project ID here." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "riG_qUokg0XZ" - }, - "outputs": [], - "source": [ - "if PROJECT_ID == \"\" or PROJECT_ID is None:\n", - " PROJECT_ID = \"\" # @param {type:\"string\"}" - ] - }, { "cell_type": "markdown", "metadata": { "id": "q7tcBkCDI1_M" }, "source": [ - "#### Timestamp\n", + "### Random ID\n", "\n", - "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a timestamp for each instance session, and append it onto the name of resources you create in this tutorial." + "To avoid name collisions between users on resources created, create a random ID for each instance session, and append the id onto the name of resources you create in this tutorial." ] }, { @@ -333,84 +433,10 @@ }, "outputs": [], "source": [ - "from datetime import datetime\n", + "import random\n", + "import string\n", "\n", - "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "t6Ggbb4DI6by" - }, - "source": [ - "### Authenticate your Google Cloud account\n", - "\n", - "**If you are using a Vertex AI Workbench notebook**, your environment is already\n", - "authenticated. Skip this step." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "RpIzUmpOI9G7" - }, - "source": [ - "**If you are using Colab**, run the cell below and follow the instructions\n", - "when prompted to authenticate your account via oAuth.\n", - "\n", - "**Otherwise**, follow these steps:\n", - "\n", - "1. In the Cloud Console, go to the [**Create service account key**\n", - " page](https://console.cloud.google.com/apis/credentials/serviceaccountkey).\n", - "\n", - "2. Click **Create service account**.\n", - "\n", - "3. In the **Service account name** field, enter a name, and\n", - " click **Create**.\n", - "\n", - "4. In the **Grant this service account access to project** section, click the **Role** drop-down list. Type \"Vertex AI\"\n", - "into the filter box, and select\n", - " **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n", - "\n", - "5. Click *Create*. A JSON file that contains your key downloads to your\n", - "local environment.\n", - "\n", - "6. Enter the path to your service account key as the\n", - "`GOOGLE_APPLICATION_CREDENTIALS` variable in the cell below and run the cell." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "AW9vQHeoI-q_" - }, - "outputs": [], - "source": [ - "import os\n", - "import sys\n", - "\n", - "# If you are running this notebook in Colab, run this cell and follow the\n", - "# instructions to authenticate your GCP account. This provides access to your\n", - "# Cloud Storage bucket and lets you submit training jobs and prediction\n", - "# requests.\n", - "\n", - "# The Vertex AI Workbench notebook product has specific requirements\n", - "IS_VERTEX_AI_WORKBENCH_NOTEBOOK = os.path.exists(\n", - " \"/opt/deeplearning/metadata/env_version\"\n", - ")\n", - "\n", - "# If on a Vertex AI Workbench notebook, then don't execute this code\n", - "if not IS_VERTEX_AI_WORKBENCH_NOTEBOOK:\n", - " if \"google.colab\" in sys.modules:\n", - " from google.colab import auth as google_auth\n", - "\n", - " google_auth.authenticate_user()\n", - "\n", - " # If you are running this notebook locally, log in using gcloud\n", - " elif not os.getenv(\"IS_TESTING\"):\n", - " ! gcloud auth login" + "RANDOM_ID = \"\".join(random.choices(string.ascii_lowercase + string.digits, k=8))" ] }, { @@ -453,7 +479,7 @@ "outputs": [], "source": [ "if BUCKET_URI == \"\" or BUCKET_URI is None or BUCKET_URI == \"gs://[your-bucket-name]\":\n", - " BUCKET_URI = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP\n", + " BUCKET_URI = \"gs://\" + PROJECT_ID + \"aip-\" + RANDOM_ID\n", "\n", "if REGION == \"[your-region]\":\n", " REGION = \"us-central1\"" @@ -528,6 +554,15 @@ "import h5py" ] }, + { + "cell_type": "markdown", + "metadata": { + "id": "76f7b9ffde0b" + }, + "source": [ + "Use gcloud to retrieve the project number." + ] + }, { "cell_type": "code", "execution_count": null, @@ -748,12 +783,10 @@ ] }, { - "cell_type": "code", - "execution_count": null, + "cell_type": "markdown", "metadata": { "id": "0f1a9fbecabb" }, - "outputs": [], "source": [ "Using the resource name, you can retrieve an existing MatchingEngineIndex." ] @@ -766,7 +799,7 @@ }, "outputs": [], "source": [ - "tree_ah_index = aiplatform.MatchingEngineIndex(INDEX_RESOURCE_NAME)" + "tree_ah_index = aiplatform.MatchingEngineIndex(index_name=INDEX_RESOURCE_NAME)" ] }, { @@ -821,7 +854,7 @@ "outputs": [], "source": [ "brute_force_index = aiplatform.MatchingEngineIndex(\n", - " \"projects/1012616486416/locations/us-central1/indexes/6738176690918260736\"\n", + " index_name=INDEX_BRUTE_FORCE_RESOURCE_NAME\n", ")" ] }, @@ -932,8 +965,9 @@ }, "outputs": [], "source": [ - "VPC_NETWORK_NAME = \"projects/{}/global/networks/{}\".format(PROJECT_NUMBER, NETWORK_NAME)\n", - "VPC_NETWORK_NAME" + "VPC_NETWORK = \"[your-network-name]\"\n", + "VPC_NETWORK_FULL = \"projects/{}/global/networks/{}\".format(PROJECT_NUMBER, VPC_NETWORK)\n", + "VPC_NETWORK_FULL" ] }, { @@ -947,7 +981,7 @@ "my_index_endpoint = aiplatform.MatchingEngineIndexEndpoint.create(\n", " display_name=\"index_endpoint_for_demo\",\n", " description=\"index endpoint description\",\n", - " network=VPC_NETWORK_NAME,\n", + " network=VPC_NETWORK_FULL,\n", ")" ] }, @@ -989,7 +1023,7 @@ }, "outputs": [], "source": [ - "DEPLOYED_INDEX_ID = f\"tree_ah_glove_deployed_{TIMESTAMP}\"" + "DEPLOYED_INDEX_ID = f\"tree_ah_glove_deployed_{RANDOM_ID}\"" ] }, { @@ -1024,7 +1058,7 @@ }, "outputs": [], "source": [ - "DEPLOYED_BRUTE_FORCE_INDEX_ID = f\"glove_brute_force_deployed_{TIMESTAMP}\"" + "DEPLOYED_BRUTE_FORCE_INDEX_ID = f\"glove_brute_force_deployed_{RANDOM_ID}\"" ] }, { @@ -1171,8 +1205,8 @@ "outputs": [], "source": [ "# Delete indexes\n", - "tree_ah_index.delete(force=True)\n", - "brute_force_index.delete(force=True)" + "tree_ah_index.delete()\n", + "brute_force_index.delete()" ] } ],