xiaohei/model_garden
1
0
Fork 0
mirror of https://github.com/GoogleCloudPlatform/vertex-ai-samples.git synced 2026-09-26 14:42:04 +00:00
Code Issues Packages Projects Releases Wiki Activity

Migrated matching engine notebook to official and added VPC network support to CI (#836)

Browse Source 
* Added matching engine notebook official

* Ran linter

* Added matching engine to .cloud-build/test_notebook_vm.txt

Co-authored-by: Andrew Ferlitsch <aferlitsch@google.com>
This commit is contained in:
Ivan Cheung
2022-08-11 18:30:22 -07:00
committed by GitHub
co-authored by Andrew Ferlitsch
parent ea36f5c43e
commit 2885a7a70f
6 changed files with 263 additions and 199 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.cloud-build/execute_changed_notebooks_cli.py
+9 -2
View File
@@ -68,6 +68,12 @@ parser.add_argument(
help="A service account. This is used to inject a variable value into the notebook before running. This is not the account that will run the notebook.",
required=True,
)
parser.add_argument(
"--variable_vpc_network",
type=str,
help="The full VPC network name. See https://cloud.google.com/compute/docs/networks-and-firewalls#networks. Format is projects/{project}/global/networks/{network}, where {project} is a project number, as in '12345', and {network} is network name. See <https://cloud.google.com/compute/docs/reference/rest/v1/networks/insert> for details. This is used to inject a variable value into the notebook before running.",
required=False,
)
parser.add_argument(
"--staging_bucket",
type=str,
@@ -114,10 +120,11 @@ execute_changed_notebooks_helper.process_and_execute_notebooks(
container_uri=args.container_uri,
staging_bucket=args.staging_bucket,
artifacts_bucket=args.artifacts_bucket,
should_parallelize=args.should_parallelize,
timeout=args.timeout,
variable_project_id=args.variable_project_id,
variable_region=args.variable_region,
variable_service_account=args.variable_service_account,
variable_vpc_network=args.variable_vpc_network,
private_pool_id=args.private_pool_id,
should_parallelize=args.should_parallelize,
timeout=args.timeout,
)
.cloud-build/execute_changed_notebooks_helper.py
+31 -9
View File
@@ -67,7 +67,7 @@ class NotebookExecutionResult:
output_uri: str
build_id: str
error_message: Optional[str]
@property
def output_uri_web(self) -> Optional[str]:
if self.output_uri.startswith("gs://"):
@@ -81,6 +81,7 @@ def _process_notebook(
variable_project_id: str,
variable_region: str,
variable_service_account: str,
variable_vpc_network: Optional[str],
):
# Read notebook
with open(notebook_path) as f:
@@ -93,6 +94,7 @@ def _process_notebook(
"PROJECT_ID": variable_project_id,
"REGION": variable_region,
"SERVICE_ACCOUNT": variable_service_account,
"VPC_NETWORK": variable_vpc_network,
},
)
@@ -128,8 +130,9 @@ def process_and_execute_notebook(
variable_project_id: str,
variable_region: str,
variable_service_account: str,
variable_vpc_network: Optional[str],
private_pool_id: Optional[str],
deadline: datetime,
deadline: datetime.datetime,
notebook: str,
should_get_tail_logs: bool = False,
) -> NotebookExecutionResult:
@@ -137,6 +140,13 @@ def process_and_execute_notebook(
print(f"Running notebook: {notebook}")
# Handle empty strings
if not variable_vpc_network:
variable_vpc_network = None
if not private_pool_id:
private_pool_id = None
# Create paths
notebook_output_uri = "/".join([artifacts_bucket, pathlib.Path(notebook).name])
@@ -163,6 +173,7 @@ def process_and_execute_notebook(
variable_project_id=variable_project_id,
variable_region=variable_region,
variable_service_account=variable_service_account,
variable_vpc_network=variable_vpc_network,
)
# Upload the pre-processed code to a GCS bucket
@@ -266,8 +277,8 @@ def get_changed_notebooks(
notebooks = []
else:
print(f"Looking for all notebooks.")
notebooks = subprocess.check_output(["git", "ls-files"] + test_paths)
notebooks = notebooks.decode("utf-8").split("\n")
notebooks_str = subprocess.check_output(["git", "ls-files"] + test_paths)
notebooks = notebooks_str.decode("utf-8").split("\n")
notebooks = [notebook for notebook in notebooks if notebook.endswith(".ipynb")]
notebooks = [notebook for notebook in notebooks if len(notebook) > 0]
@@ -286,12 +297,13 @@ def process_and_execute_notebooks(
container_uri: str,
staging_bucket: str,
artifacts_bucket: str,
should_parallelize: bool,
timeout: int,
variable_project_id: str,
variable_region: str,
variable_service_account: str,
private_pool_id: Optional[str],
should_parallelize: bool,
timeout: int,
variable_vpc_network: Optional[str] = None,
private_pool_id: Optional[str] = None,
):
"""
Run the notebooks that exist under the folders defined in the test_paths_file.
@@ -349,6 +361,7 @@ def process_and_execute_notebooks(
variable_project_id,
variable_region,
variable_service_account,
variable_vpc_network,
private_pool_id,
deadline,
),
@@ -364,6 +377,7 @@ def process_and_execute_notebooks(
variable_project_id=variable_project_id,
variable_region=variable_region,
variable_service_account=variable_service_account,
variable_vpc_network=variable_vpc_network,
private_pool_id=private_pool_id,
deadline=deadline,
notebook=notebook,
@@ -389,11 +403,18 @@ def process_and_execute_notebooks(
format_timedelta(result.duration),
result.log_url,
result.output_uri,
result.output_uri_web
result.output_uri_web,
]
for result in results_sorted
],
headers=["build_tag", "status", "duration", "log_url", "output_uri", "output_uri_web"],
headers=[
"build_tag",
"status",
"duration",
"log_url",
"output_uri",
"output_uri_web",
],
)
)
@@ -422,6 +443,7 @@ def process_and_execute_notebooks(
variable_project_id=variable_project_id,
variable_region=variable_region,
variable_service_account=variable_service_account,
variable_vpc_network=variable_vpc_network,
)
execute_notebook_helper.execute_notebook(
.cloud-build/notebook-execution-test-cloudbuild.yaml
+1 -1
View File
@@ -36,7 +36,7 @@ steps:
- -c
- |
. workspace/env/bin/activate &&
python3 .cloud-build/execute_changed_notebooks_cli.py --test_paths_file "${_TEST_PATHS_FILE}" --base_branch "${_FORCED_BASE_BRANCH}" --container_uri ${_PYTHON_IMAGE} --staging_bucket ${_GCS_STAGING_BUCKET} --artifacts_bucket ${_GCS_STAGING_BUCKET}/executed_notebooks/PR_${_PR_NUMBER}/BUILD_${BUILD_ID} --variable_project_id ${PROJECT_ID} --variable_region ${_GCP_REGION} --variable_service_account ${_GCP_SERVICE_ACCOUNT} `if [ ! -z "${_PRIVATE_POOL_NAME}" ]; then echo "--private_pool_id ${_PRIVATE_POOL_NAME}"; fi`
python3 .cloud-build/execute_changed_notebooks_cli.py --test_paths_file "${_TEST_PATHS_FILE}" --base_branch "${_FORCED_BASE_BRANCH}" --container_uri ${_PYTHON_IMAGE} --staging_bucket ${_GCS_STAGING_BUCKET} --artifacts_bucket ${_GCS_STAGING_BUCKET}/executed_notebooks/PR_${_PR_NUMBER}/BUILD_${BUILD_ID} --variable_project_id ${PROJECT_ID} --variable_region ${_GCP_REGION} --variable_service_account ${_GCP_SERVICE_ACCOUNT} --variable_vpc_network "${_GPC_VPC_NETWORK_NAME}" `if [ ! -z "${_PRIVATE_POOL_NAME}" ]; then echo "--private_pool_id ${_PRIVATE_POOL_NAME}"; fi`
env:
- 'IS_TESTING=1'
timeout: 86400s
.cloud-build/test_notebook_vm.txt
+1
View File
@@ -2,3 +2,4 @@ notebooks/official/vizier/gapic-vizier-multi-objective-optimization.ipynb
notebooks/official/pipelines/lightweight_functions_component_io_kfp.ipynb
notebooks/official/ml_metadata/sdk-metric-parameter-tracking-for-locally-trained-models.ipynb
notebooks/official/pipelines/metrics_viz_run_compare_kfp.ipynb
notebooks/official/matching_engine/sdk_matching_engine_for_indexing.ipynb
notebooks/official/CODEOWNERS
+1 -1
View File
@@ -5,7 +5,7 @@
* @GoogleCloudPlatform/vertex-ai-samples-contributors @GoogleCloudPlatform/caiis-tw
# matching_engine folder
/matching_engine @shenzhimo2
/matching_engine @shenzhimo2 @ivanmkc
/tabnet/tabnet_vertex_tutorial.ipynb @longtle
notebooks/community/matching_engine/sdk_matching_engine_for_indexing.ipynb → notebooks/official/matching_engine/sdk_matching_engine_for_indexing.ipynb
+220 -186
View File
@@ -32,13 +32,13 @@
"<table align=\"left\">\n",
"\n",
" <td>\n",
" <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/matching_engine/sdk_matching_engine_for_indexing.ipynb\">\n",
" <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/official/matching_engine/sdk_matching_engine_for_indexing.ipynb\">\n",
" <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
" Run in Vertex Workbench\n",
" </a>\n",
" </td>\n",
" <td>\n",
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/matching_engine/sdk_matching_engine_for_indexing.ipynb\">\n",
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/matching_engine/sdk_matching_engine_for_indexing.ipynb\">\n",
" <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
" View on GitHub\n",
" </a>\n",
@@ -95,8 +95,182 @@
"id": "S5zc4kbEiYCm"
},
"source": [
"## Before you begin\n",
"## Before you begin"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "d1e95a984673"
},
"source": [
"### Set up your Google Cloud project\n",
"\n",
"**The following steps are required, regardless of your notebook environment.**\n",
"\n",
"1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager).\n",
"\n",
"1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
"\n",
"1. [Enable the Vertex AI API and Compute Engine API, and Service Networking API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,compute_component,servicenetworking.googleapis.com).\n",
"\n",
"1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
"Cloud SDK uses the right project for all the commands in this notebook.\n",
"\n",
"**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "2b9daa35336a"
},
"source": [
"### Authenticate your Google Cloud account\n",
"\n",
"**If you are using a Vertex AI Workbench notebook**, your environment is already\n",
"authenticated. Skip this step."
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "c6bed8c6a6b3"
},
"source": [
"**If you are using Colab**, run the cell below and follow the instructions\n",
"when prompted to authenticate your account via oAuth.\n",
"\n",
"**Otherwise**, follow these steps:\n",
"\n",
"1. In the Cloud Console, go to the [**Create service account key**\n",
" page](https://console.cloud.google.com/apis/credentials/serviceaccountkey).\n",
"\n",
"2. Click **Create service account**.\n",
"\n",
"3. In the **Service account name** field, enter a name, and\n",
" click **Create**.\n",
"\n",
"4. In the **Grant this service account access to project** section, click the **Role** drop-down list. Type \"Vertex AI\"\n",
"into the filter box, and select\n",
" **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
"\n",
"5. Click *Create*. A JSON file that contains your key downloads to your\n",
"local environment.\n",
"\n",
"6. Enter the path to your service account key as the\n",
"`GOOGLE_APPLICATION_CREDENTIALS` variable in the cell below and run the cell."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "3e2b43c2d2bf"
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Your browser has been opened to visit:\n",
"\n",
" https://accounts.google.com/o/oauth2/auth?response_type=code&client_id=32555940559.apps.googleusercontent.com&redirect_uri=http%3A%2F%2Flocalhost%3A8085%2F&scope=openid+https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fuserinfo.email+https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fcloud-platform+https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fappengine.admin+https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fsqlservice.login+https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fcompute+https%3A%2F%2Fwww.googleapis.com%2Fauth%2Faccounts.reauth&state=UY9jjYfhoSedWWUOWXp5Pmicq0Ic04&access_type=offline&code_challenge=OQefcewSwkT7ZwfzzOVidtngvZspdY1NgN6rltw8x7A&code_challenge_method=S256\n",
"\n"
]
}
],
"source": [
"import os\n",
"import sys\n",
"\n",
"# If you are running this notebook in Colab, run this cell and follow the\n",
"# instructions to authenticate your GCP account. This provides access to your\n",
"# Cloud Storage bucket and lets you submit training jobs and prediction\n",
"# requests.\n",
"\n",
"# The Vertex AI Workbench notebook product has specific requirements\n",
"IS_VERTEX_AI_WORKBENCH_NOTEBOOK = os.path.exists(\n",
" \"/opt/deeplearning/metadata/env_version\"\n",
")\n",
"\n",
"# If on a Vertex AI Workbench notebook, then don't execute this code\n",
"if not IS_VERTEX_AI_WORKBENCH_NOTEBOOK:\n",
" if \"google.colab\" in sys.modules:\n",
" from google.colab import auth as google_auth\n",
"\n",
" google_auth.authenticate_user()\n",
"\n",
" # If you are running this notebook locally, log in using gcloud\n",
" elif not os.getenv(\"IS_TESTING\"):\n",
" ! gcloud auth login"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "WReHDGG5g0XY"
},
"source": [
"#### Set your project ID\n",
"\n",
"**If you don't know your project ID**, you may be able to get your project ID using `gcloud`."
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {
"id": "beb72f394541"
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Project ID: python-docs-samples-tests\n"
]
}
],
"source": [
"import os\n",
"\n",
"PROJECT_ID = \"[your-project-id]\" # @param {type:\"string\"}\n",
"\n",
"# Get your Google Cloud project ID from gcloud\n",
"if not os.getenv(\"IS_TESTING\"):\n",
" shell_output = !gcloud config list --format 'value(core.project)' 2>/dev/null\n",
" PROJECT_ID = shell_output[0]\n",
" print(\"Project ID: \", PROJECT_ID)"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "f4c6d0a9e66c"
},
"source": [
"Otherwise, set your project ID here."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "1dc3fa9ac4f7"
},
"outputs": [],
"source": [
"if PROJECT_ID == \"\" or PROJECT_ID is None:\n",
" PROJECT_ID = \"<your_project_id>\" # @param {type:\"string\"}"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "4962667eec8e"
},
"source": [
"* **Prepare a VPC network**. To reduce any network overhead that might lead to unnecessary increase in overhead latency, it is best to call the ANN endpoints from your VPC via a direct [VPC Peering](https://cloud.google.com/vertex-ai/docs/general/vpc-peering) connection. \n",
" * The following section describes how to setup a VPC Peering connection if you don't have one. \n",
" * This is a one-time initial setup task. You can also reuse existing VPC network and skip this section."
@@ -110,9 +284,7 @@
},
"outputs": [],
"source": [
"PROJECT_ID = \"python-docs-samples-tests\" # @param {type:\"string\"}\n",
"\n",
"NETWORK_NAME = \"ann-vpc-network\" # @param {type:\"string\"}\n",
"VPC_NETWORK = \"[your-vpc-network-name]\" # @param {type:\"string\"}\n",
"\n",
"PEERING_RANGE_NAME = \"ann-haystack-range\""
]
@@ -125,24 +297,28 @@
},
"outputs": [],
"source": [
"# Create a VPC network\n",
"! gcloud compute networks create {NETWORK_NAME} --bgp-routing-mode=regional --subnet-mode=auto --project={PROJECT_ID}\n",
"import os\n",
"\n",
"# Add necessary firewall rules\n",
"! gcloud compute firewall-rules create {NETWORK_NAME}-allow-icmp --network {NETWORK_NAME} --priority 65534 --project {PROJECT_ID} --allow icmp\n",
"# Remove the if condition to run the encapsulated code\n",
"if not os.getenv(\"IS_TESTING\"):\n",
" # Create a VPC network\n",
" ! gcloud compute networks create {VPC_NETWORK} --bgp-routing-mode=regional --subnet-mode=auto --project={PROJECT_ID}\n",
"\n",
"! gcloud compute firewall-rules create {NETWORK_NAME}-allow-internal --network {NETWORK_NAME} --priority 65534 --project {PROJECT_ID} --allow all --source-ranges 10.128.0.0/9\n",
" # Add necessary firewall rules\n",
" ! gcloud compute firewall-rules create {VPC_NETWORK}-allow-icmp --network {VPC_NETWORK} --priority 65534 --project {PROJECT_ID} --allow icmp\n",
"\n",
"! gcloud compute firewall-rules create {NETWORK_NAME}-allow-rdp --network {NETWORK_NAME} --priority 65534 --project {PROJECT_ID} --allow tcp:3389\n",
" ! gcloud compute firewall-rules create {VPC_NETWORK}-allow-internal --network {VPC_NETWORK} --priority 65534 --project {PROJECT_ID} --allow all --source-ranges 10.128.0.0/9\n",
"\n",
"! gcloud compute firewall-rules create {NETWORK_NAME}-allow-ssh --network {NETWORK_NAME} --priority 65534 --project {PROJECT_ID} --allow tcp:22\n",
" ! gcloud compute firewall-rules create {VPC_NETWORK}-allow-rdp --network {VPC_NETWORK} --priority 65534 --project {PROJECT_ID} --allow tcp:3389\n",
"\n",
"# Reserve IP range\n",
"! gcloud compute addresses create {PEERING_RANGE_NAME} --global --prefix-length=16 --network={NETWORK_NAME} --purpose=VPC_PEERING --project={PROJECT_ID} --description=\"peering range\"\n",
" ! gcloud compute firewall-rules create {VPC_NETWORK}-allow-ssh --network {VPC_NETWORK} --priority 65534 --project {PROJECT_ID} --allow tcp:22\n",
"\n",
"# Set up peering with service networking\n",
"# Your account must have the \"Compute Network Admin\" role to run the following.\n",
"! gcloud services vpc-peerings connect --service=servicenetworking.googleapis.com --network={NETWORK_NAME} --ranges={PEERING_RANGE_NAME} --project={PROJECT_ID}"
" # Reserve IP range\n",
" ! gcloud compute addresses create {PEERING_RANGE_NAME} --global --prefix-length=16 --network={VPC_NETWORK} --purpose=VPC_PEERING --project={PROJECT_ID} --description=\"peering range\"\n",
"\n",
" # Set up peering with service networking\n",
" # Your account must have the \"Compute Network Admin\" role to run the following.\n",
" ! gcloud services vpc-peerings connect --service=servicenetworking.googleapis.com --network={VPC_NETWORK} --ranges={PEERING_RANGE_NAME} --project={PROJECT_ID}"
]
},
{
@@ -230,9 +406,6 @@
},
"outputs": [],
"source": [
"# Automatically restart kernel after installs\n",
"import os\n",
"\n",
"if not os.getenv(\"IS_TESTING\"):\n",
" # Automatically restart kernel after installs\n",
" import IPython\n",
@@ -241,88 +414,15 @@
" app.kernel.do_shutdown(True)"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "BF1j6f9HApxa"
},
"source": [
"### Set up your Google Cloud project\n",
"\n",
"**The following steps are required, regardless of your notebook environment.**\n",
"\n",
"1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager).\n",
"\n",
"1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
"\n",
"1. [Enable the Vertex AI API and Compute Engine API, and Service Networking API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,compute_component,servicenetworking.googleapis.com).\n",
"\n",
"1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
"Cloud SDK uses the right project for all the commands in this notebook.\n",
"\n",
"**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "WReHDGG5g0XY"
},
"source": [
"#### Set your project ID\n",
"\n",
"**If you don't know your project ID**, you may be able to get your project ID using `gcloud`."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "oM1iC_MfAts1"
},
"outputs": [],
"source": [
"import os\n",
"\n",
"PROJECT_ID = \"python-docs-samples-tests\"\n",
"\n",
"# Get your Google Cloud project ID from gcloud\n",
"if not os.getenv(\"IS_TESTING\"):\n",
" shell_output = !gcloud config list --format 'value(core.project)' 2>/dev/null\n",
" PROJECT_ID = shell_output[0]\n",
" print(\"Project ID: \", PROJECT_ID)"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "qJYoRfYng0XZ"
},
"source": [
"Otherwise, set your project ID here."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "riG_qUokg0XZ"
},
"outputs": [],
"source": [
"if PROJECT_ID == \"\" or PROJECT_ID is None:\n",
" PROJECT_ID = \"<your_project_id>\" # @param {type:\"string\"}"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "q7tcBkCDI1_M"
},
"source": [
"#### Timestamp\n",
"### Random ID\n",
"\n",
"If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a timestamp for each instance session, and append it onto the name of resources you create in this tutorial."
"To avoid name collisions between users on resources created, create a random ID for each instance session, and append the id onto the name of resources you create in this tutorial."
]
},
{
@@ -333,84 +433,10 @@
},
"outputs": [],
"source": [
"from datetime import datetime\n",
"import random\n",
"import string\n",
"\n",
"TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "t6Ggbb4DI6by"
},
"source": [
"### Authenticate your Google Cloud account\n",
"\n",
"**If you are using a Vertex AI Workbench notebook**, your environment is already\n",
"authenticated. Skip this step."
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "RpIzUmpOI9G7"
},
"source": [
"**If you are using Colab**, run the cell below and follow the instructions\n",
"when prompted to authenticate your account via oAuth.\n",
"\n",
"**Otherwise**, follow these steps:\n",
"\n",
"1. In the Cloud Console, go to the [**Create service account key**\n",
" page](https://console.cloud.google.com/apis/credentials/serviceaccountkey).\n",
"\n",
"2. Click **Create service account**.\n",
"\n",
"3. In the **Service account name** field, enter a name, and\n",
" click **Create**.\n",
"\n",
"4. In the **Grant this service account access to project** section, click the **Role** drop-down list. Type \"Vertex AI\"\n",
"into the filter box, and select\n",
" **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
"\n",
"5. Click *Create*. A JSON file that contains your key downloads to your\n",
"local environment.\n",
"\n",
"6. Enter the path to your service account key as the\n",
"`GOOGLE_APPLICATION_CREDENTIALS` variable in the cell below and run the cell."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "AW9vQHeoI-q_"
},
"outputs": [],
"source": [
"import os\n",
"import sys\n",
"\n",
"# If you are running this notebook in Colab, run this cell and follow the\n",
"# instructions to authenticate your GCP account. This provides access to your\n",
"# Cloud Storage bucket and lets you submit training jobs and prediction\n",
"# requests.\n",
"\n",
"# The Vertex AI Workbench notebook product has specific requirements\n",
"IS_VERTEX_AI_WORKBENCH_NOTEBOOK = os.path.exists(\n",
" \"/opt/deeplearning/metadata/env_version\"\n",
")\n",
"\n",
"# If on a Vertex AI Workbench notebook, then don't execute this code\n",
"if not IS_VERTEX_AI_WORKBENCH_NOTEBOOK:\n",
" if \"google.colab\" in sys.modules:\n",
" from google.colab import auth as google_auth\n",
"\n",
" google_auth.authenticate_user()\n",
"\n",
" # If you are running this notebook locally, log in using gcloud\n",
" elif not os.getenv(\"IS_TESTING\"):\n",
" ! gcloud auth login"
"RANDOM_ID = \"\".join(random.choices(string.ascii_lowercase + string.digits, k=8))"
]
},
{
@@ -453,7 +479,7 @@
"outputs": [],
"source": [
"if BUCKET_URI == \"\" or BUCKET_URI is None or BUCKET_URI == \"gs://[your-bucket-name]\":\n",
" BUCKET_URI = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP\n",
" BUCKET_URI = \"gs://\" + PROJECT_ID + \"aip-\" + RANDOM_ID\n",
"\n",
"if REGION == \"[your-region]\":\n",
" REGION = \"us-central1\""
@@ -528,6 +554,15 @@
"import h5py"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "76f7b9ffde0b"
},
"source": [
"Use gcloud to retrieve the project number."
]
},
{
"cell_type": "code",
"execution_count": null,
@@ -748,12 +783,10 @@
]
},
{
"cell_type": "code",
"execution_count": null,
"cell_type": "markdown",
"metadata": {
"id": "0f1a9fbecabb"
},
"outputs": [],
"source": [
"Using the resource name, you can retrieve an existing MatchingEngineIndex."
]
@@ -766,7 +799,7 @@
},
"outputs": [],
"source": [
"tree_ah_index = aiplatform.MatchingEngineIndex(INDEX_RESOURCE_NAME)"
"tree_ah_index = aiplatform.MatchingEngineIndex(index_name=INDEX_RESOURCE_NAME)"
]
},
{
@@ -821,7 +854,7 @@
"outputs": [],
"source": [
"brute_force_index = aiplatform.MatchingEngineIndex(\n",
" \"projects/1012616486416/locations/us-central1/indexes/6738176690918260736\"\n",
" index_name=INDEX_BRUTE_FORCE_RESOURCE_NAME\n",
")"
]
},
@@ -932,8 +965,9 @@
},
"outputs": [],
"source": [
"VPC_NETWORK_NAME = \"projects/{}/global/networks/{}\".format(PROJECT_NUMBER, NETWORK_NAME)\n",
"VPC_NETWORK_NAME"
"VPC_NETWORK = \"[your-network-name]\"\n",
"VPC_NETWORK_FULL = \"projects/{}/global/networks/{}\".format(PROJECT_NUMBER, VPC_NETWORK)\n",
"VPC_NETWORK_FULL"
]
},
{
@@ -947,7 +981,7 @@
"my_index_endpoint = aiplatform.MatchingEngineIndexEndpoint.create(\n",
" display_name=\"index_endpoint_for_demo\",\n",
" description=\"index endpoint description\",\n",
" network=VPC_NETWORK_NAME,\n",
" network=VPC_NETWORK_FULL,\n",
")"
]
},
@@ -989,7 +1023,7 @@
},
"outputs": [],
"source": [
"DEPLOYED_INDEX_ID = f\"tree_ah_glove_deployed_{TIMESTAMP}\""
"DEPLOYED_INDEX_ID = f\"tree_ah_glove_deployed_{RANDOM_ID}\""
]
},
{
@@ -1024,7 +1058,7 @@
},
"outputs": [],
"source": [
"DEPLOYED_BRUTE_FORCE_INDEX_ID = f\"glove_brute_force_deployed_{TIMESTAMP}\""
"DEPLOYED_BRUTE_FORCE_INDEX_ID = f\"glove_brute_force_deployed_{RANDOM_ID}\""
]
},
{
@@ -1171,8 +1205,8 @@
"outputs": [],
"source": [
"# Delete indexes\n",
"tree_ah_index.delete(force=True)\n",
"brute_force_index.delete(force=True)"
"tree_ah_index.delete()\n",
"brute_force_index.delete()"
]
}
],