mirror of
https://github.com/GoogleCloudPlatform/vertex-ai-samples.git
synced 2026-09-26 22:51:56 +00:00
Compare commits
3
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Andrew Ferlitsch
|
08419d9c4b | ||
|
Andrew Ferlitsch
|
7efbb0adf6 | ||
|
Andrew Ferlitsch
|
4f5b330af3 |
@@ -0,0 +1,906 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "Z3U--8tt1BOd"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Copyright 2023 Google LLC\n",
|
||||
"#\n",
|
||||
"# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
|
||||
"# you may not use this file except in compliance with the License.\n",
|
||||
"# You may obtain a copy of the License at\n",
|
||||
"#\n",
|
||||
"# https://www.apache.org/licenses/LICENSE-2.0\n",
|
||||
"#\n",
|
||||
"# Unless required by applicable law or agreed to in writing, software\n",
|
||||
"# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
|
||||
"# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
|
||||
"# See the License for the specific language governing permissions and\n",
|
||||
"# limitations under the License."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "b5368ecbbc71"
|
||||
},
|
||||
"source": [
|
||||
"# TODO\n",
|
||||
"\n",
|
||||
"<table align=\"left\">\n",
|
||||
" <td>\n",
|
||||
" <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/automl_tabular_on_vertex_pipelines.ipynb\">\n",
|
||||
" <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
" <td>\n",
|
||||
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/automl_forecasting_on_vertex_pipelines.ipynb\">\n",
|
||||
" <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
|
||||
" View on GitHub\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
" <td>\n",
|
||||
" <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/official/automl/automl_forecasting_on_vertex_pipelines.ipynb\">\n",
|
||||
" <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
|
||||
" Open in Vertex AI Workbench\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
"</table>\n",
|
||||
"<br/><br/><br/>"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "ks3kQQs9uWuA"
|
||||
},
|
||||
"source": [
|
||||
"# Onboarding Instructions"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "gMF4pbEIuZS2"
|
||||
},
|
||||
"source": [
|
||||
"Send a email to warehouse-trusted-testers-mailing-external+managers@google.com asking to join the group and allowlist for the project. Please mention the project id in the email.\n",
|
||||
"\n",
|
||||
"Onboarding documentation see [here](https://docs.google.com/document/d/1sPI5WQUgq9s8raAkxl3k00kxh3PhKyxIhOn8X1GNmw8/edit?usp=sharing)\n",
|
||||
"\n",
|
||||
"Colab CUJ see [here](https://docs.google.com/document/d/1sPI5WQUgq9s8raAkxl3k00kxh3PhKyxIhOn8X1GNmw8/edit#bookmark=id.1t2o3wjj4lss)\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "e85f0288a6df"
|
||||
},
|
||||
"source": [
|
||||
"## Install additional packages\n",
|
||||
"\n",
|
||||
"Install the Google Cloud Pipeline Components (GCPC) SDK not earlier than `2.3.0`.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "iZj1BqoTxKNN"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# @title Download SDK\n",
|
||||
"! gsutil cp gs://visionai-private-artifacts/f54638dfe40e579744dd9b7d59dfce85e813891e/visionai-0.0.5-py3-none-any.whl .\n",
|
||||
"\n",
|
||||
"# @title Install SDK\n",
|
||||
"# You can restart the runtime if it asks for.\n",
|
||||
"! pip3 install visionai-0.0.5-py3-none-any.whl --force-reinstall\n",
|
||||
"! pip3 install absl-py"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "Bj5O0S5RTxzY"
|
||||
},
|
||||
"source": [
|
||||
"### Colab only: Uncomment the following cell to restart the kernel."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "023DMKUaTypt"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Automatically restart kernel after installs so that your environment can access the new packages\n",
|
||||
"# import IPython\n",
|
||||
"\n",
|
||||
"# app = IPython.Application.instance()\n",
|
||||
"# app.kernel.do_shutdown(True)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "yfEglUHQk9S3"
|
||||
},
|
||||
"source": [
|
||||
"## Before you begin\n",
|
||||
"\n",
|
||||
"### Set up your Google Cloud project\n",
|
||||
"\n",
|
||||
"**The following steps are required, regardless of your notebook environment.**\n",
|
||||
"\n",
|
||||
"1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
|
||||
"\n",
|
||||
"2. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
|
||||
"\n",
|
||||
"3. [Enable the Vertex AI API: Vertex AI APIs, Dataflow APIs, Compute Engine APIs, and Cloud Storage](https://console.cloud.google.com/flows/enableapi?apiid=ml.googleapis.com,dataflow.googleapis.com,compute_component,storage-component.googleapis.com).\n",
|
||||
"\n",
|
||||
"4. If you are running this notebook locally, you need to install the [Cloud SDK](https://cloud.google.com/sdk)."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "95cb7ffd6895"
|
||||
},
|
||||
"source": [
|
||||
"### Set your project ID\n",
|
||||
"\n",
|
||||
"**If you don't know your project ID**, try the following:\n",
|
||||
"* Run `gcloud config list`.\n",
|
||||
"* Run `gcloud projects list`.\n",
|
||||
"* See the support page: [Locate the project ID](https://support.google.com/googleapi/answer/7014113)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "cd85f5c794e5"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"PROJECT_ID = \"[your-project-id]\" # @param {type:\"string\"}\n",
|
||||
"\n",
|
||||
"# Set the project id\n",
|
||||
"! gcloud config set project {PROJECT_ID}"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "b12f508d97c6"
|
||||
},
|
||||
"source": [
|
||||
"### Region\n",
|
||||
"\n",
|
||||
"You can also change the `REGION` variable used by Vertex AI. Learn more about [Vertex AI regions](https://cloud.google.com/vertex-ai/docs/general/locations)."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "8e8b7997de7a"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"REGION = \"us-central1\" # @param {type: \"string\"}"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "eu0e2TRVxjHb"
|
||||
},
|
||||
"source": [
|
||||
"### Authenticate your Google Cloud account\n",
|
||||
"\n",
|
||||
"Depending on your Jupyter environment, you may have to manually authenticate. Follow the relevant instructions below."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "d118c95af93f"
|
||||
},
|
||||
"source": [
|
||||
"**1. Vertex AI Workbench**\n",
|
||||
"* Do nothing as you are already authenticated."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "3035286fcdda"
|
||||
},
|
||||
"source": [
|
||||
"**2. Local JupyterLab instance, uncomment and run:**"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "455882ec0f11"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# ! gcloud auth login"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "5097f3233d53"
|
||||
},
|
||||
"source": [
|
||||
"**3. Colab, uncomment and run:**"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "2b88e46ac2c8"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# from google.colab import auth\n",
|
||||
"# auth.authenticate_user()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "fcdbb8929927"
|
||||
},
|
||||
"source": [
|
||||
"**4. Service account or other**\n",
|
||||
"* See how to grant Cloud Storage permissions to your service account at https://cloud.google.com/storage/docs/gsutil/commands/iam#ch-examples."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "ixfp-_Ne09EK"
|
||||
},
|
||||
"source": [
|
||||
"## Set Up Variables"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "_6ePddpJiGIb"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Please run the whole section after changing any value of the variables.\n",
|
||||
"PROJECT_NUMBER_STR = !gcloud projects describe $PROJECT_ID --format=\"value(projectNumber)\"\n",
|
||||
"PROJECT_NUMBER = int(PROJECT_NUMBER_STR[0])\n",
|
||||
"\n",
|
||||
"# Only us-central1 is supported.\n",
|
||||
"# Please note that this region is for VisionAi services. For speech\n",
|
||||
"# transcription, we may not respect the region here.\n",
|
||||
"LOCATION_ID = REGION\n",
|
||||
"\n",
|
||||
"CORPUS_DISPLAY_NAME = \"Demo corpus\" # @param {type: \"string\"}\n",
|
||||
"CORPUS_DESCRIPTION = \"Demo corpus to demo warehouse transformations and search\" # @param {type: \"string\"}\n",
|
||||
"# If CORPUS_ID is specified, skip creating a new corpus.\n",
|
||||
"CORPUS_ID = None # @param {type: \"string\"}\n",
|
||||
"\n",
|
||||
"# External users can only access PROD environment.\n",
|
||||
"ENV = \"PROD\" # @param {type: \"string\"}\n",
|
||||
"\n",
|
||||
"# You can also create cluster via UI by creating a stream. Setting CLUSTER_ID\n",
|
||||
"# None and USE_EXISTING_CLUSTER to True will use the one created from UI.\n",
|
||||
"USE_EXISTING_CLUSTER = False # @param {type: \"boolean\"}\n",
|
||||
"CLUSTER_ID = \"application-cluster-0\" # @param {type: \"string\"}\n",
|
||||
"\n",
|
||||
"# If DEPLOYED_INDEX_ID is specified, use existing index instead of creating and\n",
|
||||
"# deploying a new index.\n",
|
||||
"DEPLOYED_INDEX_ID = None # @param {type: \"string\"}\n",
|
||||
"INDEX_DISPLAY_NAME = \"Demo Index\" # @param {type: \"string\"}\n",
|
||||
"INDEX_ENDPOINT_DISPLAY_NAME = \"Demo Index Endpoint\" # @param {type: \"string\"}\n",
|
||||
"\n",
|
||||
"CLEAN_UP_ASSETS = False # @param {type: \"boolean\"}\n",
|
||||
"CLEAN_UP_INDEX = False # @param {type: \"boolean\"}\n",
|
||||
"CLEAN_UP_CORPUS = False # @param {type: \"boolean\"}\n",
|
||||
"CLEAN_UP_CLUSTER = False # @param {type: \"boolean\"}"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "nYgf32fogKHa"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Files to be processed.\n",
|
||||
"GCS_FILES = [\n",
|
||||
" \"gs://cloud-samples-data/video/animals.mp4\",\n",
|
||||
" \"gs://cloud-samples-data/video/googlework_short.mp4\",\n",
|
||||
" \"gs://cloud-samples-data/video/chicago.mp4\",\n",
|
||||
" (\n",
|
||||
" \"gs://cloud-samples-data/video/Machine Learning Solving Problems\"\n",
|
||||
" \" Big, Small, and Prickly.mp4\"\n",
|
||||
" ),\n",
|
||||
" \"gs://cloud-samples-data/video/JaneGoodall.mp4\",\n",
|
||||
" \"gs://cloud-samples-data/video/gbikes_dinosaur.mp4\",\n",
|
||||
" \"gs://cloud-samples-data/video/pizza.mp4\",\n",
|
||||
"]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "PPHprqn0-npv"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from absl import flags\n",
|
||||
"\n",
|
||||
"try:\n",
|
||||
" if FLAGS_IS_DEFINED:\n",
|
||||
" print(\"Flags have been defined.\")\n",
|
||||
"except:\n",
|
||||
" _PROJECT_NUMBER = flags.DEFINE_integer(\"project_number\", None, \"Project number.\")\n",
|
||||
" _LOCATION_ID = flags.DEFINE_string(\"location_id\", \"us-central1\", \"Location id.\")\n",
|
||||
" _CORPUS_DISPLAY_NAME = flags.DEFINE_string(\n",
|
||||
" \"corpus_display_name\", \"Demo Corpus\", \"Corpus display name.\"\n",
|
||||
" )\n",
|
||||
" _CORPUS_DISCRIPTION = flags.DEFINE_string(\n",
|
||||
" \"corpus_description\",\n",
|
||||
" \"Demo Corpus to interact with warehouse\",\n",
|
||||
" \"Corpus description.\",\n",
|
||||
" )\n",
|
||||
" _CORPUS_ID = flags.DEFINE_string(\n",
|
||||
" \"corpus_id\", None, \"If specified, use existing VOD corpus.\"\n",
|
||||
" )\n",
|
||||
" _GCS_FILES = flags.DEFINE_list(\n",
|
||||
" \"gcs_files\",\n",
|
||||
" [\n",
|
||||
" \"gs://cloud-samples-data/video/animals.mp4\",\n",
|
||||
" \"gs://cloud-samples-data/video/googlework_short.mp4\",\n",
|
||||
" \"gs://cloud-samples-data/video/chicago.mp4\",\n",
|
||||
" (\n",
|
||||
" \"gs://cloud-samples-data/video/Machine Learning Solving Problems\"\n",
|
||||
" \" Big, Small, and Prickly.mp4\"\n",
|
||||
" ),\n",
|
||||
" \"gs://cloud-samples-data/video/JaneGoodall.mp4\",\n",
|
||||
" ],\n",
|
||||
" \"GCS files.\",\n",
|
||||
" )\n",
|
||||
" _ENV = flags.DEFINE_enum(\n",
|
||||
" \"env\",\n",
|
||||
" \"PROD\",\n",
|
||||
" [\"AUTOPUSH\", \"STAGING\", \"PROD\"],\n",
|
||||
" \"The environment.\",\n",
|
||||
" )\n",
|
||||
" _USE_EXISTING_CLUSTER = flags.DEFINE_bool(\n",
|
||||
" \"use_existing_cluster\",\n",
|
||||
" False,\n",
|
||||
" \"Whether create a new cluster. If this is false, and cluster flag is None,\"\n",
|
||||
" \" it will use the cluster created from UI .\",\n",
|
||||
" )\n",
|
||||
" _CLUSTER_ID = flags.DEFINE_string(\n",
|
||||
" \"cluster_id\",\n",
|
||||
" None,\n",
|
||||
" \"The cluster. If not specified, by default using the cluster created\"\n",
|
||||
" \" via UI.\",\n",
|
||||
" )\n",
|
||||
" _DEPLOYED_INDEX_ID = flags.DEFINE_string(\n",
|
||||
" \"deployed_index_id\",\n",
|
||||
" None,\n",
|
||||
" \"If specified, use existing index instead of creating and deploying a new\"\n",
|
||||
" \" one.\",\n",
|
||||
" )\n",
|
||||
" _INDEX_DISPLAY_NAME = flags.DEFINE_string(\n",
|
||||
" \"index_display_name\", \"Demo Index\", \"Index display name.\"\n",
|
||||
" )\n",
|
||||
" _INDEX_ENDPOINT_DISPLAY_NAME = flags.DEFINE_string(\n",
|
||||
" \"index_endpoint_display_name\",\n",
|
||||
" \"Demo index endpoint\",\n",
|
||||
" \"Display name for the index endpoint.\",\n",
|
||||
" )\n",
|
||||
" _CLEAN_UP_ASSETS = flags.DEFINE_bool(\n",
|
||||
" \"clean_up_assets\", False, \"Whether clean up assets.\"\n",
|
||||
" )\n",
|
||||
" _CLEAN_UP_CORPUS = flags.DEFINE_bool(\n",
|
||||
" \"clean_up_corpus\", False, \"Whether clean up corpus.\"\n",
|
||||
" )\n",
|
||||
" _CLEAN_UP_INDEX = flags.DEFINE_bool(\n",
|
||||
" \"clean_up_index\", False, \"Whether clean up index and index endpoint.\"\n",
|
||||
" )\n",
|
||||
" _CLEAN_UP_CLUSTER = flags.DEFINE_bool(\n",
|
||||
" \"clean_up_cluster\", False, \"Whether clean up cluster.\"\n",
|
||||
" )\n",
|
||||
" FLAGS_IS_DEFINED = True"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "VcJWIPOL0mXx"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"FLAGS = flags.FLAGS\n",
|
||||
"\n",
|
||||
"FLAGS[\"project_number\"].parse(PROJECT_NUMBER)\n",
|
||||
"FLAGS[\"location_id\"].parse(LOCATION_ID)\n",
|
||||
"FLAGS[\"corpus_display_name\"].parse(CORPUS_DISPLAY_NAME)\n",
|
||||
"FLAGS[\"corpus_description\"].parse(CORPUS_DESCRIPTION)\n",
|
||||
"if CORPUS_ID:\n",
|
||||
" FLAGS[\"corpus_id\"].parse(CORPUS_ID)\n",
|
||||
"else:\n",
|
||||
" FLAGS[\"corpus_id\"].unparse()\n",
|
||||
"FLAGS[\"gcs_files\"].parse(GCS_FILES)\n",
|
||||
"FLAGS[\"env\"].parse(ENV)\n",
|
||||
"FLAGS[\"use_existing_cluster\"].parse(USE_EXISTING_CLUSTER)\n",
|
||||
"if CLUSTER_ID:\n",
|
||||
" FLAGS[\"cluster_id\"].parse(CLUSTER_ID)\n",
|
||||
"else:\n",
|
||||
" FLAGS[\"cluster_id\"].unparse()\n",
|
||||
"if DEPLOYED_INDEX_ID:\n",
|
||||
" FLAGS[\"deployed_index_id\"].parse(DEPLOYED_INDEX_ID)\n",
|
||||
"else:\n",
|
||||
" FLAGS[\"deployed_index_id\"].unparse()\n",
|
||||
"FLAGS[\"index_display_name\"].parse(INDEX_DISPLAY_NAME)\n",
|
||||
"FLAGS[\"index_endpoint_display_name\"].parse(INDEX_ENDPOINT_DISPLAY_NAME)\n",
|
||||
"FLAGS[\"clean_up_assets\"].parse(CLEAN_UP_ASSETS)\n",
|
||||
"FLAGS[\"clean_up_index\"].parse(CLEAN_UP_INDEX)\n",
|
||||
"FLAGS[\"clean_up_corpus\"].parse(CLEAN_UP_CORPUS)\n",
|
||||
"FLAGS[\"clean_up_cluster\"].parse(CLEAN_UP_CLUSTER)\n",
|
||||
"\n",
|
||||
"FLAGS.mark_as_parsed()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "VxXIz2fhpbe3"
|
||||
},
|
||||
"source": [
|
||||
"## Enable API"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "PBV1hcV_spkd"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"!gcloud services enable videointelligence.googleapis.com"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "soFPlRaso3rs"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def get_service_endpoint(env: str) -> str:\n",
|
||||
" if env == \"STAGING\":\n",
|
||||
" return \"staging-visionai.sandbox.googleapis.com\"\n",
|
||||
" if env == \"AUTOPUSH\":\n",
|
||||
" return \"autopush-visionai.sandbox.googleapis.com\"\n",
|
||||
" return \"visionai.googleapis.com\"\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"visionai_service_endpoint = get_service_endpoint(ENV)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "shwQ0FmCBzim"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"!gcloud services enable {visionai_service_endpoint}"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "pBaITbuI_8c-"
|
||||
},
|
||||
"source": [
|
||||
"# Example"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "tgumkNAeoBGg"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# @title Config logging\n",
|
||||
"import logging\n",
|
||||
"\n",
|
||||
"logging.basicConfig()\n",
|
||||
"logging.getLogger().setLevel(logging.INFO)\n",
|
||||
"_logger = logging.getLogger(\"colab\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "b74iOAaMynQT"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# @title Imports\n",
|
||||
"import concurrent\n",
|
||||
"import logging\n",
|
||||
"\n",
|
||||
"from absl import flags\n",
|
||||
"from visionai.python.gapic.visionai import visionai_v1\n",
|
||||
"from visionai.python.net import channel\n",
|
||||
"from visionai.python.streams import client as streams_client\n",
|
||||
"from visionai.python.warehouse.transformer import \\\n",
|
||||
" asset_indexing_transformer as ait\n",
|
||||
"from visionai.python.warehouse.transformer import (ocr_transformer,\n",
|
||||
" speech_transformer,\n",
|
||||
" transformer_factory)\n",
|
||||
"from visionai.python.warehouse.utils import (vod_asset, vod_corpus,\n",
|
||||
" vod_index_endpoint)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "EjBTqlAricqz"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# @title Creates a warehouse client to talk with warehouse.\n",
|
||||
"warehouse_endpoint = channel.get_warehouse_service_endpoint(\n",
|
||||
" channel.Environment[_ENV.value]\n",
|
||||
")\n",
|
||||
"warehouse_client = visionai_v1.WarehouseClient(\n",
|
||||
" client_options={\"api_endpoint\": warehouse_endpoint}\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "wl1wprgNf0cz"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# @title Creates a cluster.\n",
|
||||
"if not _USE_EXISTING_CLUSTER.value:\n",
|
||||
" if _CLUSTER_ID.value is None:\n",
|
||||
" raise ValueError(\"Cluster must be specified when creating new cluster.\")\n",
|
||||
" streams_client.create_cluster(\n",
|
||||
" channel.ConnectionOptions(\n",
|
||||
" _PROJECT_NUMBER.value,\n",
|
||||
" _LOCATION_ID.value,\n",
|
||||
" _CLUSTER_ID.value,\n",
|
||||
" channel.Environment[_ENV.value],\n",
|
||||
" )\n",
|
||||
" )"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "oEVMGRjuf5oJ"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# @title Creates a corpus or use existing corpus.\n",
|
||||
"if _CORPUS_ID.value is None:\n",
|
||||
" corpus_name = vod_corpus.create_corpus(\n",
|
||||
" warehouse_client,\n",
|
||||
" _PROJECT_NUMBER.value,\n",
|
||||
" _LOCATION_ID.value,\n",
|
||||
" _CORPUS_DISPLAY_NAME.value,\n",
|
||||
" _CORPUS_DISCRIPTION.value,\n",
|
||||
" ).name\n",
|
||||
"else:\n",
|
||||
" corpus_name = visionai_v1.WarehouseClient.corpus_path(\n",
|
||||
" _PROJECT_NUMBER.value, _LOCATION_ID.value, _CORPUS_ID.value\n",
|
||||
" )"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "YXh9TlJv9Ayz"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# @title Creates a executor to upload and transform assets in parallel.\n",
|
||||
"executor = concurrent.futures.ThreadPoolExecutor(max_workers=8)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "jZWKPBR_yHIF"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# @title Creates and Uploads Assets\n",
|
||||
"new_asset_futures = []\n",
|
||||
"for gcs_file in _GCS_FILES.value:\n",
|
||||
" new_asset_futures.append(\n",
|
||||
" executor.submit(\n",
|
||||
" vod_asset.create_and_upload_asset,\n",
|
||||
" warehouse_client,\n",
|
||||
" gcs_file,\n",
|
||||
" corpus_name,\n",
|
||||
" )\n",
|
||||
" )\n",
|
||||
"done_or_error, _ = concurrent.futures.wait(\n",
|
||||
" new_asset_futures, return_when=\"ALL_COMPLETED\"\n",
|
||||
")\n",
|
||||
"asset_names = []\n",
|
||||
"for done_future in done_or_error:\n",
|
||||
" try:\n",
|
||||
" asset_names.append(done_future.result())\n",
|
||||
" _logger.info(\"Create and upload asset succeeded %s\", done_future.result())\n",
|
||||
" except Exception as e:\n",
|
||||
" _logger.exception(e)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "DJpefxMzl4FM"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# @title Create index and index endpoint for the corpus, or use existing index\n",
|
||||
"# and index endpoint if specified.\n",
|
||||
"if _DEPLOYED_INDEX_ID.value is None:\n",
|
||||
" # Creates index for the corpus.\n",
|
||||
" index_name = vod_corpus.index_corpus(\n",
|
||||
" warehouse_client, corpus_name, _INDEX_DISPLAY_NAME.value\n",
|
||||
" )\n",
|
||||
" # Creates index endpoint and deploys the created index above to the index\n",
|
||||
" # endpoint.\n",
|
||||
" index_endpoint_name = vod_index_endpoint.create_index_endpoint(\n",
|
||||
" warehouse_client,\n",
|
||||
" _PROJECT_NUMBER.value,\n",
|
||||
" _LOCATION_ID.value,\n",
|
||||
" _INDEX_ENDPOINT_DISPLAY_NAME.value,\n",
|
||||
" ).name\n",
|
||||
" deploy_operation = warehouse_client.deploy_index(\n",
|
||||
" visionai_v1.DeployIndexRequest(\n",
|
||||
" index_endpoint=index_endpoint_name,\n",
|
||||
" deployed_index=visionai_v1.DeployedIndex(\n",
|
||||
" index=index_name,\n",
|
||||
" ),\n",
|
||||
" )\n",
|
||||
" )\n",
|
||||
" _logger.info(\"Wait for index to be deployed %s.\", deploy_operation.operation.name)\n",
|
||||
" # Wait for the deploy index operation. Depends on the data size to be\n",
|
||||
" # indexed, the timeout may need to be increased.\n",
|
||||
" deploy_operation.result(timeout=7200)\n",
|
||||
" _logger.info(\"Index is deployed.\")\n",
|
||||
"else:\n",
|
||||
" index_name = \"{}/indexes/{}\".format(corpus_name, _DEPLOYED_INDEX_ID.value)\n",
|
||||
" index = warehouse_client.get_index(visionai_v1.GetIndexRequest(name=index_name))\n",
|
||||
" _logger.info(\"Use existing index %s.\", index)\n",
|
||||
" if index.state != visionai_v1.Index.State.CREATED:\n",
|
||||
" _logger.critical(\"Invalid index. The index state must be Created.\")\n",
|
||||
" if not index.deployed_indexes:\n",
|
||||
" _logger.critical(\"Invalid index. The index must be deployed.\")\n",
|
||||
" index_endpoint_name = index.deployed_indexes[0].index_endpoint"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "LPp5LV0-x4ZS"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# @title Run Transforms\n",
|
||||
"# If you run into errors like \"Resource '.../analyses/ocr-warehouse-text-lang\n",
|
||||
"# was not found.\", please set variable USE_EXISTING_CLUSTER to False, it will\n",
|
||||
"# create the cluster.\n",
|
||||
"ocr_config = ocr_transformer.OcrTransformerInitConfig(\n",
|
||||
" corpus_name=corpus_name,\n",
|
||||
" env=channel.Environment[_ENV.value],\n",
|
||||
")\n",
|
||||
"if _CLUSTER_ID.value:\n",
|
||||
" ocr_config.cluster_id = _CLUSTER_ID.value\n",
|
||||
"ml_config = transformer_factory.MlTransformersCreationConfig(\n",
|
||||
" run_embedding=True,\n",
|
||||
" speech_transformer_init_config=speech_transformer.SpeechTransformerInitConfig(\n",
|
||||
" corpus_name=corpus_name, language_code=\"en-US\"\n",
|
||||
" ),\n",
|
||||
" ocr_transformer_init_config=ocr_config,\n",
|
||||
")\n",
|
||||
"ml_transformers = transformer_factory.create_ml_transformers(\n",
|
||||
" warehouse_client, ml_config\n",
|
||||
")\n",
|
||||
"# Creates indexing transformer to index assets.\n",
|
||||
"asset_indexing_transformer = ait.AssetIndexingTransformer(warehouse_client, index_name)\n",
|
||||
"# Runs the transformers for the assets.\n",
|
||||
"futures = []\n",
|
||||
"\n",
|
||||
"for asset_name in asset_names:\n",
|
||||
" futures.append(\n",
|
||||
" executor.submit(\n",
|
||||
" vod_asset.transform_single_asset,\n",
|
||||
" asset_name,\n",
|
||||
" ml_transformers,\n",
|
||||
" asset_indexing_transformer,\n",
|
||||
" )\n",
|
||||
" )\n",
|
||||
"done_or_error, _ = concurrent.futures.wait(futures, return_when=\"ALL_COMPLETED\")\n",
|
||||
"for future in done_or_error:\n",
|
||||
" try:\n",
|
||||
" future.result()\n",
|
||||
" except Exception as e:\n",
|
||||
" _logger.exception(e)\n",
|
||||
"\n",
|
||||
"all_transformers = ml_transformers + [asset_indexing_transformer]\n",
|
||||
"for transformer in all_transformers:\n",
|
||||
" transformer.teardown()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "fKlKYnA7hEPT"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# @title Search\n",
|
||||
"search_response = warehouse_client.search_index_endpoint(\n",
|
||||
" visionai_v1.SearchIndexEndpointRequest(\n",
|
||||
" index_endpoint=index_endpoint_name,\n",
|
||||
" text_query=\"dinosaur\",\n",
|
||||
" page_size=10,\n",
|
||||
" )\n",
|
||||
")\n",
|
||||
"_logger.info(\"Search response: %s\", search_response)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "S0znp7rq8PvS"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"cr = visionai_v1.Criteria(\n",
|
||||
" field=\"speech\", text_array=visionai_v1.StringArray(txt_values=[\"kid\"])\n",
|
||||
")\n",
|
||||
"search_response = warehouse_client.search_index_endpoint(\n",
|
||||
" visionai_v1.SearchIndexEndpointRequest(\n",
|
||||
" index_endpoint=index_endpoint_name,\n",
|
||||
" text_query=\"river\",\n",
|
||||
" criteria=[cr],\n",
|
||||
" page_size=100,\n",
|
||||
" )\n",
|
||||
")\n",
|
||||
"_logger.info(\"Search response: %s\", search_response)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "9ypuAtP0E8y-"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"cr = visionai_v1.Criteria(\n",
|
||||
" field=\"text\", text_array=visionai_v1.StringArray(txt_values=[\"National Park\"])\n",
|
||||
")\n",
|
||||
"search_response = warehouse_client.search_index_endpoint(\n",
|
||||
" visionai_v1.SearchIndexEndpointRequest(\n",
|
||||
" index_endpoint=index_endpoint_name,\n",
|
||||
" text_query=\"trees\",\n",
|
||||
" criteria=[cr],\n",
|
||||
" page_size=100,\n",
|
||||
" )\n",
|
||||
")\n",
|
||||
"_logger.info(\"Search response: %s\", search_response)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "VZt1pXWfsaAt"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# @title Clean up\n",
|
||||
"if _CLEAN_UP_ASSETS.value:\n",
|
||||
" for asset_name in asset_names:\n",
|
||||
" warehouse_client.delete_asset(visionai_v1.DeleteAssetRequest(name=asset_name))\n",
|
||||
" _logger.info(\"Deleted asset %s\", asset_name)\n",
|
||||
"\n",
|
||||
"if _CLEAN_UP_INDEX.value:\n",
|
||||
" undeploy_operation = warehouse_client.undeploy_index(\n",
|
||||
" visionai_v1.UndeployIndexRequest(index_endpoint=index_endpoint_name)\n",
|
||||
" )\n",
|
||||
" _logger.info(\n",
|
||||
" \"Wait for index to be undeployed %s.\",\n",
|
||||
" undeploy_operation.operation.name,\n",
|
||||
" )\n",
|
||||
" # Wait for the undeploy index operation.\n",
|
||||
" undeploy_operation.result(timeout=1800)\n",
|
||||
" _logger.info(\"Index is undeployed.\")\n",
|
||||
" warehouse_client.delete_index(visionai_v1.DeleteIndexRequest(name=index_name))\n",
|
||||
" _logger.info(\"Deleted index %s\", index_name)\n",
|
||||
" warehouse_client.delete_index_endpoint(\n",
|
||||
" visionai_v1.DeleteIndexEndpointRequest(name=index_endpoint_name)\n",
|
||||
" )\n",
|
||||
" _logger.info(\"Deleted index endpoint %s\", index_endpoint_name)\n",
|
||||
"\n",
|
||||
"if _CLEAN_UP_CORPUS.value:\n",
|
||||
" warehouse_client.delete_corpus(visionai_v1.DeleteCorpusRequest(name=corpus_name))\n",
|
||||
" _logger.info(\"Deleted corpus %s\", corpus_name)\n",
|
||||
"\n",
|
||||
"if _CLEAN_UP_CLUSTER.value:\n",
|
||||
" if _CLUSTER_ID.value is None:\n",
|
||||
" _logger.warning(\"Can't clean up cluster since cluster_id is not specified.\")\n",
|
||||
" else:\n",
|
||||
" streams_client.delete_cluster(\n",
|
||||
" channel.ConnectionOptions(\n",
|
||||
" _PROJECT_NUMBER.value,\n",
|
||||
" _LOCATION_ID.value,\n",
|
||||
" _CLUSTER_ID.value,\n",
|
||||
" channel.Environment[_ENV.value],\n",
|
||||
" ),\n",
|
||||
" )"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"colab": {
|
||||
"name": "Video_Warehouse_CUJ_Demo_(Private_Preview).ipynb",
|
||||
"toc_visible": true
|
||||
},
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3",
|
||||
"name": "python3"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 0
|
||||
}
|
||||
Reference in New Issue
Block a user