Compare commits

..
603 changed files with 18825 additions and 34344 deletions
@@ -365,7 +365,7 @@ def process_and_execute_notebook(
# Use gcloud to get tail
try:
result.error_message = subprocess.check_output(
["gcloud", "storage", "cat", "--range", "-1000", log_file_uri], encoding="UTF-8"
["gsutil", "cat", "-r", "-1000", log_file_uri], encoding="UTF-8"
)
except Exception as error:
result.error_message = str(error)
+2 -2
View File
@@ -56,8 +56,8 @@ def execute_notebook(
print("\n=== DOWNLOAD EXECUTED NOTEBOOK ===\n")
print(f"Please debug the executed notebook by downloading the executed notebook:")
print("Option 1. Using gcloud storage. Run the following command in your terminal.")
print(f'\tgcloud storage cp "{output_file_or_uri}" .')
print("Option 1. Using gsutil. Run the following command in your terminal.")
print(f'\tgsutil cp "{output_file_or_uri}" .')
print("Option 2. Using this link.")
print(f"\thttps://storage.googleapis.com/{output_file_or_uri[5:]}")
+1 -1
View File
@@ -108,7 +108,7 @@ class VertexAIInstallProprocessor(Preprocessor):
if "google-cloud-aiplatform" not in content:
return content
return (
f"gcloud storage cp {self.vertex_ai_wheel} google-cloud-aiplatform.whl\n" +
f"gsutil cp {self.vertex_ai_wheel} google-cloud-aiplatform.whl\n" +
content.replace("google-cloud-aiplatform\n", "google-cloud-aiplatform.whl\n")
.replace("google-cloud-aiplatform ", "google-cloud-aiplatform.whl ")
)
+2 -2
View File
@@ -15,7 +15,7 @@ def download_file(bucket_name: str, blob_name: str, destination_file: str) -> st
remote_file_path = "".join(["gs://", "/".join([bucket_name, blob_name])])
subprocess.check_output(
["gcloud", "storage", "cp", remote_file_path, destination_file], encoding="UTF-8"
["gsutil", "cp", remote_file_path, destination_file], encoding="UTF-8"
)
return destination_file
@@ -27,7 +27,7 @@ def upload_file(
) -> str:
"""Copies a local file to a GCS path"""
subprocess.check_output(
["gcloud", "storage", "cp", local_file_path, remote_file_path], encoding="UTF-8"
["gsutil", "cp", local_file_path, remote_file_path], encoding="UTF-8"
)
return remote_file_path
+3 -3
View File
@@ -7,11 +7,11 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Set up Python
uses: actions/setup-python@v6
uses: actions/setup-python@v5
with:
python-version: '3.12'
python-version: '3.x'
- name: Fetch pull request branch
uses: actions/checkout@v6
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Fetch base main branch
+1 -1
View File
@@ -4,7 +4,7 @@
# 2. To lint specific notebooks:
# docker run -v ${PWD}:/setup/app gcr.io/python-docs-samples-tests/notebook_linter:latest notebooks/1.ipynb notebooks/2.ipynb
FROM python:3.14
FROM python:3.13
WORKDIR setup
+3 -3
View File
@@ -2,9 +2,9 @@ git+https://github.com/tensorflow/docs
ipython
jupyter
nbconvert
black==25.12.0
pyupgrade==3.21.2
isort==7.0.0
black==25.1.0
pyupgrade==3.20.0
isort==6.0.1
flake8==7.3.0
nbqa==1.9.1
@@ -148,7 +148,7 @@ implementation:
# Downloading the model archive from GCS
# TODO: Fix gsutil bugs (requires project ID, has auth issues) and use gsutil instead.
# gcloud storage cp "$model_archive_uri" "$model_archive_local_path"
# gsutil cp "$model_archive_uri" "$model_archive_local_path"
pip install google-cloud-storage
python -c '
import sys
@@ -24,12 +24,12 @@ implementation:
# Checking whether the URI points to a single blob, a directory or a URI pattern
# URI points to a blob when that URI does not end with slash and listing that URI only yields the same URI
if [[ "$uri" != */ ]] && (gcloud storage ls "$uri" | grep --fixed-strings --line-regexp "$uri"); then
if [[ "$uri" != */ ]] && (gsutil ls "$uri" | grep --fixed-strings --line-regexp "$uri"); then
mkdir -p "$(dirname "$output_path")"
gcloud storage cp --recursive "$uri" "$output_path"
gsutil -m cp -r "$uri" "$output_path"
else
mkdir -p "$output_path" # When source path is a directory, gsutil requires the destination to also be a directory
gcloud storage rsync --recursive "$uri" "$output_path" # gsutil cp has different path handling than Linux cp. It always puts the source directory (name) inside the destination directory. gsutil rsync does not have that problem.
gsutil -m rsync -r "$uri" "$output_path" # gsutil cp has different path handling than Linux cp. It always puts the source directory (name) inside the destination directory. gsutil rsync does not have that problem.
fi
- inputValue: GCS path
- outputPath: Data
@@ -1,3 +1,3 @@
torch==2.8.0
torch==2.2.0
torchvision==0.9.1
tensorboard==2.5.0
@@ -110,7 +110,7 @@
},
"outputs": [],
"source": [
"! gcloud storage ls $gcs_output_uri_prefix"
"! gsutil ls $gcs_output_uri_prefix"
]
},
{
@@ -192,7 +192,7 @@
},
"outputs": [],
"source": [
"! gcloud storage cp --recursive $gcs_output_uri_prefix/model ./model_server/"
"! gsutil cp -r $gcs_output_uri_prefix/model ./model_server/"
]
},
{
@@ -556,7 +556,7 @@
},
"outputs": [],
"source": [
"! gcloud storage rm --recursive --continue-on-error $gcs_output_uri_prefix"
"! gsutil rm -rf $gcs_output_uri_prefix"
]
},
{
@@ -412,7 +412,7 @@
},
"outputs": [],
"source": [
"! gcloud storage ls $gcs_output_uri_prefix"
"! gsutil ls $gcs_output_uri_prefix"
]
}
],
@@ -77,4 +77,4 @@ echo "After the job is completed successfully, model files will be saved at $JOB
# # Verify the model was exported
# echo "Verify the model was exported:"
# gcloud storage ls ${JOB_DIR}/
# gsutil ls ${JOB_DIR}/
@@ -34,4 +34,4 @@ RUN echo "service_envelope=json\n" "inference_address=http://0.0.0.0:${AIP_H
USER model-server
# run Torchserve HTTP serve to respond to prediction requests
CMD ["echo", "AIP_STORAGE_URI=${AIP_STORAGE_URI}", ";", "gcloud", "storage", "cp", "--recursive", "${AIP_STORAGE_URI}/${MODEL_NAME}.mar", "/home/model-server/model-store/", ";", "ls", "-ltr", "/home/model-server/model-store/", ";", "torchserve", "--start", "--ts-config=/home/model-server/config.properties", "--models", "${MODEL_NAME}=${MODEL_NAME}.mar", "--model-store", "/home/model-server/model-store"]
CMD ["echo", "AIP_STORAGE_URI=${AIP_STORAGE_URI}", ";", "gsutil", "cp", "-r", "${AIP_STORAGE_URI}/${MODEL_NAME}.mar", "/home/model-server/model-store/", ";", "ls", "-ltr", "/home/model-server/model-store/", ";", "torchserve", "--start", "--ts-config=/home/model-server/config.properties", "--models", "${MODEL_NAME}=${MODEL_NAME}.mar", "--model-store", "/home/model-server/model-store"]
@@ -67,4 +67,4 @@ echo "After the job is completed successfully, model files will be saved at $JOB
# # Verify the model was exported
# echo "Verify the model was exported:"
# gcloud storage ls ${JOB_DIR}/
# gsutil ls ${JOB_DIR}/
@@ -478,7 +478,8 @@
},
"outputs": [],
"source": [
"! gcloud storage buckets create --location $REGION $BUCKET_NAME" ]
"! gsutil mb -l $REGION $BUCKET_NAME"
]
},
{
"cell_type": "markdown",
@@ -497,7 +498,8 @@
},
"outputs": [],
"source": [
"! gcloud storage ls --all-versions --long $BUCKET_NAME" ]
"! gsutil ls -al $BUCKET_NAME"
]
},
{
"cell_type": "markdown",
@@ -580,7 +582,8 @@
"outputs": [],
"source": [
"# Download the sample data into your RAW_DATA_PATH\n",
"! gcloud storage cp \"gs://cloud-samples-data/vertex-ai/community-content/tf_agents_bandits_movie_recommendation_with_kfp_and_vertex_sdk/u.data\" $RAW_DATA_PATH" ]
"! gsutil cp \"gs://cloud-samples-data/vertex-ai/community-content/tf_agents_bandits_movie_recommendation_with_kfp_and_vertex_sdk/u.data\" $RAW_DATA_PATH"
]
},
{
"cell_type": "code",
@@ -1618,7 +1621,9 @@
"! gcloud scheduler jobs delete $SIMULATOR_SCHEDULER_JOB --quiet\n",
"\n",
"# Delete Cloud Storage objects that were created.\n",
"! gcloud storage rm --recursive $PIPELINE_ROOT\n", "! gcloud storage rm --recursive $TRAINING_ARTIFACTS_DIR" ]
"! gsutil -m rm -r $PIPELINE_ROOT\n",
"! gsutil -m rm -r $TRAINING_ARTIFACTS_DIR"
]
}
],
"metadata": {
@@ -398,7 +398,6 @@
"if not IS_GOOGLE_CLOUD_NOTEBOOK:\n",
" if \"google.colab\" in sys.modules:\n",
" from google.colab import auth as google_auth\n",
"\n",
" google_auth.authenticate_user()\n",
"\n",
" # If you are running this notebook locally, replace the string below with the\n",
@@ -473,7 +472,7 @@
},
"outputs": [],
"source": [
"! gcloud storage buckets create --location $REGION $BUCKET_NAME"
"! gsutil mb -l $REGION $BUCKET_NAME"
]
},
{
@@ -493,7 +492,7 @@
},
"outputs": [],
"source": [
"! gcloud storage ls --all-versions --long $BUCKET_NAME"
"! gsutil ls -al $BUCKET_NAME"
]
},
{
@@ -566,7 +565,7 @@
"outputs": [],
"source": [
"# Copy the sample data into your DATA_PATH\n",
"! gcloud storage cp \"gs://cloud-samples-data/vertex-ai/community-content/tf_agents_bandits_movie_recommendation_with_kfp_and_vertex_sdk/u.data\" $DATA_PATH"
"! gsutil cp \"gs://cloud-samples-data/vertex-ai/community-content/tf_agents_bandits_movie_recommendation_with_kfp_and_vertex_sdk/u.data\" $DATA_PATH"
]
},
{
@@ -580,15 +579,11 @@
"# Set hyperparameters.\n",
"BATCH_SIZE = 8 # @param {type:\"integer\"} Training and prediction batch size.\n",
"TRAINING_LOOPS = 5 # @param {type:\"integer\"} Number of training iterations.\n",
"STEPS_PER_LOOP = (\n",
" 2 # @param {type:\"integer\"} Number of driver steps per training iteration.\n",
")\n",
"STEPS_PER_LOOP = 2 # @param {type:\"integer\"} Number of driver steps per training iteration.\n",
"\n",
"# Set MovieLens simulation environment parameters.\n",
"RANK_K = 20 # @param {type:\"integer\"} Rank for matrix factorization in the MovieLens environment; also the observation dimension.\n",
"NUM_ACTIONS = (\n",
" 20 # @param {type:\"integer\"} Number of actions (movie items) to choose from.\n",
")\n",
"NUM_ACTIONS = 20 # @param {type:\"integer\"} Number of actions (movie items) to choose from.\n",
"PER_ARM = False # Use the non-per-arm version of the MovieLens environment.\n",
"\n",
"# Set agent parameters.\n",
@@ -626,8 +621,7 @@
"source": [
"# Define RL environment.\n",
"env = movielens_py_environment.MovieLensPyEnvironment(\n",
" DATA_PATH, RANK_K, BATCH_SIZE, num_movies=NUM_ACTIONS, csv_delimiter=\"\\t\"\n",
")\n",
" DATA_PATH, RANK_K, BATCH_SIZE, num_movies=NUM_ACTIONS, csv_delimiter=\"\\t\")\n",
"environment = tf_py_environment.TFPyEnvironment(env)\n",
"\n",
"# Define RL agent/algorithm.\n",
@@ -637,8 +631,7 @@
" tikhonov_weight=TIKHONOV_WEIGHT,\n",
" alpha=AGENT_ALPHA,\n",
" dtype=tf.float32,\n",
" accepts_per_arm_features=PER_ARM,\n",
")\n",
" accepts_per_arm_features=PER_ARM)\n",
"print(\"TimeStep Spec (for each batch):\\n\", agent.time_step_spec, \"\\n\")\n",
"print(\"Action Spec (for each batch):\\n\", agent.action_spec, \"\\n\")\n",
"print(\"Reward Spec (for each batch):\\n\", environment.reward_spec(), \"\\n\")\n",
@@ -646,8 +639,7 @@
"# Define RL metric.\n",
"optimal_reward_fn = functools.partial(\n",
" environment_utilities.compute_optimal_reward_with_movielens_environment,\n",
" environment=environment,\n",
")\n",
" environment=environment)\n",
"regret_metric = tf_bandit_metrics.RegretMetric(optimal_reward_fn)\n",
"metrics = [regret_metric]"
]
@@ -712,38 +704,35 @@
" if training_data_spec_transformation_fn is None:\n",
" data_spec = agent.policy.trajectory_spec\n",
" else:\n",
" data_spec = training_data_spec_transformation_fn(agent.policy.trajectory_spec)\n",
" replay_buffer = trainer.get_replay_buffer(\n",
" data_spec, environment.batch_size, steps_per_loop\n",
" )\n",
" data_spec = training_data_spec_transformation_fn(\n",
" agent.policy.trajectory_spec)\n",
" replay_buffer = trainer.get_replay_buffer(data_spec, environment.batch_size,\n",
" steps_per_loop)\n",
"\n",
" # `step_metric` records the number of individual rounds of bandit interaction;\n",
" # that is, (number of trajectories) * batch_size.\n",
" step_metric = tf_metrics.EnvironmentSteps()\n",
" metrics = [\n",
" tf_metrics.NumberOfEpisodes(),\n",
" tf_metrics.AverageEpisodeLengthMetric(batch_size=environment.batch_size),\n",
" tf_metrics.AverageEpisodeLengthMetric(batch_size=environment.batch_size)\n",
" ]\n",
" if additional_metrics:\n",
" metrics += additional_metrics\n",
"\n",
" if isinstance(environment.reward_spec(), dict):\n",
" metrics += [\n",
" tf_metrics.AverageReturnMultiMetric(\n",
" reward_spec=environment.reward_spec(), batch_size=environment.batch_size\n",
" )\n",
" ]\n",
" metrics += [tf_metrics.AverageReturnMultiMetric(\n",
" reward_spec=environment.reward_spec(),\n",
" batch_size=environment.batch_size)]\n",
" else:\n",
" metrics += [tf_metrics.AverageReturnMetric(batch_size=environment.batch_size)]\n",
" metrics += [\n",
" tf_metrics.AverageReturnMetric(batch_size=environment.batch_size)]\n",
"\n",
" # Store intermediate metric results, indexed by metric names.\n",
" metric_results = defaultdict(list)\n",
"\n",
" if training_data_spec_transformation_fn is not None:\n",
"\n",
" def add_batch_fn(data):\n",
" return replay_buffer.add_batch(training_data_spec_transformation_fn(data))\n",
"\n",
" def add_batch_fn(data): return replay_buffer.add_batch(training_data_spec_transformation_fn(data)) \n",
" \n",
" else:\n",
" add_batch_fn = replay_buffer.add_batch\n",
"\n",
@@ -753,12 +742,10 @@
" env=environment,\n",
" policy=agent.collect_policy,\n",
" num_steps=steps_per_loop * environment.batch_size,\n",
" observers=observers,\n",
" )\n",
" observers=observers)\n",
"\n",
" training_loop = trainer.get_training_loop_fn(\n",
" driver, replay_buffer, agent, steps_per_loop\n",
" )\n",
" driver, replay_buffer, agent, steps_per_loop)\n",
" saver = policy_saver.PolicySaver(agent.policy)\n",
"\n",
" for _ in range(training_loops):\n",
@@ -796,8 +783,7 @@
" environment=environment,\n",
" training_loops=TRAINING_LOOPS,\n",
" steps_per_loop=STEPS_PER_LOOP,\n",
" additional_metrics=metrics,\n",
")\n",
" additional_metrics=metrics)\n",
"\n",
"tf.profiler.experimental.stop()"
]
@@ -1106,15 +1092,11 @@
},
"outputs": [],
"source": [
"RUN_HYPERPARAMETER_TUNING = (\n",
" True # Execute hyperparameter tuning instead of regular training.\n",
")\n",
"RUN_HYPERPARAMETER_TUNING = True # Execute hyperparameter tuning instead of regular training.\n",
"TRAIN_WITH_BEST_HYPERPARAMETERS = False # Do not train.\n",
"\n",
"HPTUNING_RESULT_DIR = \"hptuning/\" # @param {type: \"string\"} Directory to store the best hyperparameter(s) in `BUCKET_NAME` and locally (temporarily).\n",
"HPTUNING_RESULT_PATH = os.path.join(\n",
" HPTUNING_RESULT_DIR, \"result.json\"\n",
") # @param {type: \"string\"} Path to the file containing the best hyperparameter(s)."
"HPTUNING_RESULT_PATH = os.path.join(HPTUNING_RESULT_DIR, \"result.json\") # @param {type: \"string\"} Path to the file containing the best hyperparameter(s)."
]
},
{
@@ -1142,7 +1124,7 @@
" image_uri: str,\n",
" args: List[str],\n",
" location: str = \"us-central1\",\n",
" api_endpoint: str = \"us-central1-aiplatform.googleapis.com\",\n",
" api_endpoint: str = \"us-central1-aiplatform.googleapis.com\"\n",
") -> None:\n",
" \"\"\"Creates a hyperparameter tuning job using a custom container.\n",
"\n",
@@ -1215,8 +1197,8 @@
"\n",
" # Create job\n",
" response = client.create_hyperparameter_tuning_job(\n",
" parent=parent, hyperparameter_tuning_job=hyperparameter_tuning_job\n",
" )\n",
" parent=parent,\n",
" hyperparameter_tuning_job=hyperparameter_tuning_job)\n",
" job_id = response.name.split(\"/\")[-1]\n",
" print(\"Job ID:\", job_id)\n",
" print(\"Job config:\", response)\n",
@@ -1260,8 +1242,7 @@
" image_uri=f\"gcr.io/{PROJECT_ID}/{HPTUNING_TRAINING_CONTAINER}:latest\",\n",
" args=args,\n",
" location=REGION,\n",
" api_endpoint=f\"{REGION}-aiplatform.googleapis.com\",\n",
")"
" api_endpoint=f\"{REGION}-aiplatform.googleapis.com\")"
]
},
{
@@ -1311,8 +1292,7 @@
" name = client.hyperparameter_tuning_job_path(\n",
" project=project,\n",
" location=location,\n",
" hyperparameter_tuning_job=hyperparameter_tuning_job_id,\n",
" )\n",
" hyperparameter_tuning_job=hyperparameter_tuning_job_id)\n",
" response = client.get_hyperparameter_tuning_job(name=name)\n",
" return response"
]
@@ -1333,8 +1313,7 @@
" location=REGION,\n",
" api_endpoint=f\"{REGION}-aiplatform.googleapis.com\")\n",
" if response.state.name == 'JOB_STATE_SUCCEEDED':\n",
" print(\"Job succeeded.\n",
"Job Time:\", response.update_time - response.create_time)\n",
" print(\"Job succeeded.\\nJob Time:\", response.update_time - response.create_time)\n",
" trials = response.trials\n",
" print(\"Trials:\", trials)\n",
" break\n",
@@ -1369,8 +1348,8 @@
"if trials:\n",
" # Dict mapping from metric names to the best metric values seen so far\n",
" best_objective_values = dict.fromkeys(\n",
" [metric.metric_id for metric in trials[0].final_measurement.metrics], -np.inf\n",
" )\n",
" [metric.metric_id for metric in trials[0].final_measurement.metrics],\n",
" -np.inf)\n",
" # Dict mapping from metric names to a list of the best combination(s) of\n",
" # hyperparameter(s). Each combination is a dict mapping from hyperparameter\n",
" # names to their values.\n",
@@ -1379,13 +1358,12 @@
" # `final_measurement` and `parameters` are `RepeatedComposite` objects.\n",
" # Reference the structure above to extract the value of your interest.\n",
" for metric in trial.final_measurement.metrics:\n",
" params = {param.parameter_id: param.value for param in trial.parameters}\n",
" params = {\n",
" param.parameter_id: param.value for param in trial.parameters}\n",
" if metric.value > best_objective_values[metric.metric_id]:\n",
" best_params[metric.metric_id] = [params]\n",
" elif metric.value == best_objective_values[metric.metric_id]:\n",
" best_params[param.parameter_id].append(\n",
" params\n",
" ) # Handle cases where multiple hyperparameter values lead to the same performance.\n",
" best_params[param.parameter_id].append(params) # Handle cases where multiple hyperparameter values lead to the same performance.\n",
" print(\"Best hyperparameter value(s):\")\n",
" for metric, params in best_params.items():\n",
" print(f\"Metric={metric}: {sorted(params)}\")\n",
@@ -1465,9 +1443,7 @@
},
"outputs": [],
"source": [
"PREDICTION_CONTAINER = (\n",
" \"prediction-custom-container\" # @param {type:\"string\"} Name of the container image.\n",
")"
"PREDICTION_CONTAINER = \"prediction-custom-container\" # @param {type:\"string\"} Name of the container image."
]
},
{
@@ -1499,7 +1475,7 @@
" machineType: 'E2_HIGHCPU_8'\"\"\".format(\n",
" PROJECT_ID=PROJECT_ID,\n",
" PREDICTION_CONTAINER=PREDICTION_CONTAINER,\n",
" ARTIFACTS_DIR=ARTIFACTS_DIR,\n",
" ARTIFACTS_DIR=ARTIFACTS_DIR\n",
")\n",
"\n",
"with open(\"cloudbuild.yaml\", \"w\") as fp:\n",
@@ -1616,12 +1592,8 @@
},
"outputs": [],
"source": [
"RUN_HYPERPARAMETER_TUNING = (\n",
" False # Execute regular training instead of hyperparameter tuning.\n",
")\n",
"TRAIN_WITH_BEST_HYPERPARAMETERS = (\n",
" True # @param {type:\"bool\"} Whether to use learned hyperparameters in training.\n",
")"
"RUN_HYPERPARAMETER_TUNING = False # Execute regular training instead of hyperparameter tuning.\n",
"TRAIN_WITH_BEST_HYPERPARAMETERS = True # @param {type:\"bool\"} Whether to use learned hyperparameters in training."
]
},
{
@@ -1661,12 +1633,10 @@
"job = aiplatform.CustomContainerTrainingJob(\n",
" display_name=\"train-movielens\",\n",
" container_uri=f\"gcr.io/{PROJECT_ID}/{HPTUNING_TRAINING_CONTAINER}:latest\",\n",
" command=[\"python3\", \"-m\", \"src.training.task\"]\n",
" + args, # Pass in training arguments, including hyperparameters.\n",
" command=[\"python3\", \"-m\", \"src.training.task\"] + args, # Pass in training arguments, including hyperparameters.\n",
" model_serving_container_image_uri=f\"gcr.io/{PROJECT_ID}/{PREDICTION_CONTAINER}:latest\",\n",
" model_serving_container_predict_route=\"/predict\",\n",
" model_serving_container_health_route=\"/health\",\n",
")\n",
" model_serving_container_health_route=\"/health\")\n",
"\n",
"print(\"Training Spec:\", job._managed_model)\n",
"\n",
@@ -1675,8 +1645,7 @@
" replica_count=1,\n",
" machine_type=\"n1-standard-4\",\n",
" accelerator_type=\"ACCELERATOR_TYPE_UNSPECIFIED\",\n",
" accelerator_count=0,\n",
")"
" accelerator_count=0)"
]
},
{
@@ -1815,7 +1784,7 @@
"! gcloud ai models delete $model.name --quiet\n",
"\n",
"# Delete Cloud Storage objects that were created\n",
"! gcloud storage rm --recursive $ARTIFACTS_DIR"
"! gsutil -m rm -r $ARTIFACTS_DIR"
]
}
],
@@ -324,7 +324,7 @@
},
"outputs": [],
"source": [
"! gcloud storage ls $gcs_output_uri_prefix"
"! gsutil ls $gcs_output_uri_prefix"
]
},
{
@@ -344,7 +344,7 @@
},
"outputs": [],
"source": [
"! gcloud storage rm --recursive --continue-on-error $gcs_output_uri_prefix"
"! gsutil rm -rf $gcs_output_uri_prefix"
]
}
],
@@ -328,7 +328,7 @@
},
"outputs": [],
"source": [
"! gcloud storage ls $gcs_output_uri_prefix"
"! gsutil ls $gcs_output_uri_prefix"
]
},
{
@@ -348,7 +348,7 @@
},
"outputs": [],
"source": [
"! gcloud storage rm --recursive --continue-on-error $gcs_output_uri_prefix"
"! gsutil rm -rf $gcs_output_uri_prefix"
]
}
],
@@ -341,7 +341,7 @@
},
"outputs": [],
"source": [
"! gcloud storage ls $gcs_output_uri_prefix"
"! gsutil ls $gcs_output_uri_prefix"
]
},
{
@@ -361,7 +361,7 @@
},
"outputs": [],
"source": [
"! gcloud storage rm --recursive --continue-on-error $gcs_output_uri_prefix"
"! gsutil rm -rf $gcs_output_uri_prefix"
]
}
],
@@ -5,6 +5,6 @@ immutabledict==4.2.1
protobuf==4.25.8
opencv-python-headless==4.11.0.86
docutils==0.16
urllib3==2.6.3
urllib3==2.5.0
google-cloud-storage==3.0.0
retrying
@@ -45,5 +45,5 @@ six==1.17.0
sniffio==1.3.1
typing-inspection==0.4.0
typing_extensions==4.13.2
urllib3==2.6.3
urllib3==2.4.0
websockets==15.0.1
@@ -66,7 +66,7 @@ mkdir -p "$local_folder"
mkdir -p "$output_folder"
# Download the content from the GCS URI
gcloud storage cp --recursive "$gcs_dataset_path"/* "$local_folder/"
gsutil -m cp -r "$gcs_dataset_path"/* "$local_folder/"
# Process files in the local folder
for file in "$local_folder"/*; do
@@ -122,23 +122,23 @@ cp -r "$output_folder" "$images_folder"/images_2
pushd "$images_folder"/images_2
ls | xargs -P 8 -I {} mogrify -resize 50% {}
popd
gcloud storage cp --recursive "$images_folder"/images_2/* "$gcs_experiment_path"/data/images_2
gsutil -m cp -r "$images_folder"/images_2/* "$gcs_experiment_path"/data/images_2
cp -r "$output_folder" "$images_folder"/images_4
pushd "$images_folder"/images_4
ls | xargs -P 8 -I {} mogrify -resize 25% {}
popd
gcloud storage cp --recursive "$images_folder"/images_4/* "$gcs_experiment_path"/data/images_4
gsutil -m cp -r "$images_folder"/images_4/* "$gcs_experiment_path"/data/images_4
cp -r "$output_folder" "$images_folder"/images_8
pushd "$images_folder"/images_8
ls | xargs -P 8 -I {} mogrify -resize 12.5% {}
popd
gcloud storage cp "$images_folder"/images_8/* "$gcs_experiment_path"/data/images_8
gsutil -m cp "$images_folder"/images_8/* "$gcs_experiment_path"/data/images_8
# Copy images and sparse reconstruction files to gcs experiment folder.
gcloud storage cp "$images_folder"/images/* "$gcs_experiment_path"/data/images
gcloud storage cp --recursive "$local_folder"/sparse "$gcs_experiment_path"/data
gcloud storage cp "$local_folder"/database.db "$gcs_experiment_path"/data
gsutil -m cp "$images_folder"/images/* "$gcs_experiment_path"/data/images
gsutil -m cp -r "$local_folder"/sparse "$gcs_experiment_path"/data
gsutil -m cp "$local_folder"/database.db "$gcs_experiment_path"/data
echo "Processing complete."
@@ -99,14 +99,14 @@ create_dir_if_not_exists "$CHECKPOINTS_PATH"
touch "$local_experiment_path/$exp_folder_name/log_render.txt"
# Copy experiment from GCS bucket to local
gcloud storage cp --recursive "${args[-gcs_experiment_path]}/data" "$local_experiment_path/$exp_folder_name" || exit 1
gcloud storage cp --recursive "${args[-gcs_experiment_path]}/checkpoints/${training_job_name}/*" "$CHECKPOINTS_PATH" || exit 1
gsutil -m cp -r "${args[-gcs_experiment_path]}/data" "$local_experiment_path/$exp_folder_name" || exit 1
gsutil -m cp -r "${args[-gcs_experiment_path]}/checkpoints/${training_job_name}/*" "$CHECKPOINTS_PATH" || exit 1
# Check and copy keyframes file.
if [[ -n ${args[-gcs_keyframes_file]} ]]; then
keyframes_file_basename=$(basename "${args[-gcs_keyframes_file]}")
local_keyframes_file="$local_dataset_path/$keyframes_file_basename"
gcloud storage cp "${args[-gcs_keyframes_file]}" "$local_keyframes_file" || exit 1
gsutil cp "${args[-gcs_keyframes_file]}" "$local_keyframes_file" || exit 1
echo "Local keyframe file: $local_keyframes_file"
launch_rendering "$local_keyframes_file"
else
@@ -114,4 +114,4 @@ else
fi
# Copy rendered data back to GCS.
gcloud storage cp --recursive "$OUTPUT_RENDER_PATH" "${args[-gcs_experiment_path]}/render/${rendering_job_name}"
gsutil -m cp -r "$OUTPUT_RENDER_PATH" "${args[-gcs_experiment_path]}/render/${rendering_job_name}"
@@ -74,7 +74,7 @@ create_dir_if_not_exists "$local_experiment_path"
create_dir_if_not_exists "$local_experiment_path/$scene_folder_name"
# Copy experiment from GCS bucket to local.
gcloud storage cp --recursive "${gcs_experiment_path}/data" "$local_experiment_path/$scene_folder_name" || exit 1
gsutil -m cp -r "${gcs_experiment_path}/data" "$local_experiment_path/$scene_folder_name" || exit 1
echo "GCS Experiment: $gcs_experiment_path"
echo "Gin Config File: $gin_config_file"
@@ -89,6 +89,6 @@ accelerate launch train.py --gin_configs="$gin_config_file" \
--gin_bindings="Config.factor = ${factor}" \
--gin_bindings="Config.max_steps = ${max_training_steps}"
gcloud storage rm --recursive "${gcs_experiment_path}/checkpoints/${training_job_name}"
gcloud storage cp --recursive "$local_experiment_path/$scene_folder_name/config.gin" "${gcs_experiment_path}/${training_job_name}_config.gin"
gcloud storage cp --recursive "$local_experiment_path/$scene_folder_name/checkpoints/*/*" "${gcs_experiment_path}/checkpoints/${training_job_name}"
gsutil -m rm -r "${gcs_experiment_path}/checkpoints/${training_job_name}"
gsutil -m cp -r "$local_experiment_path/$scene_folder_name/config.gin" "${gcs_experiment_path}/${training_job_name}_config.gin"
gsutil -m cp -r "$local_experiment_path/$scene_folder_name/checkpoints/*/*" "${gcs_experiment_path}/checkpoints/${training_job_name}"
@@ -102,10 +102,10 @@ def download_gcs_uri_to_local(
if not os.path.exists(destination_dir):
os.mkdir(destination_dir)
subprocess.check_output([
"gcloud",
"storage",
"gsutil",
"-m",
"cp",
"--recursive",
"-r",
gcs_uri,
destination_dir,
])
-2
View File
@@ -26,8 +26,6 @@
/vertex_endpoints/find_ideal_machine_type/find_ideal_machine_type/find_ideal_machine_type.ipynb @entrpn
/vertex_endpoints/nvidia-triton/nvidia-triton-custom-container-prediction.ipynb @RajeshThallam
/vertex_endpoints/optimized_tensorflow_runtime @vlasenkoalexey
/notebooks/community/alphagenome/cloudai_alphagenome_vai_quickstart.ipynb @dpanigra
/notebooks/community/weathernext/weathernext_2_early_access_program.ipynb @dpanigra
/notebooks/community/ml_ops/stage2/get_started_with_visionapi_and_automl.ipynb @mansari
/notebooks/community/neo4j/graph_paysim.ipynb @benofben @laeg
/notebooks/community/ml_ops/stage1/get_started_with_visionapi_and_vertex_datasets.ipynb @mansari
@@ -1,552 +0,0 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "f705f4be70e9"
},
"outputs": [],
"source": [
"# Copyright 2025 Google LLC\n",
"#\n",
"# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
"# you may not use this file except in compliance with the License.\n",
"# You may obtain a copy of the License at\n",
"#\n",
"# https://www.apache.org/licenses/LICENSE-2.0\n",
"#\n",
"# Unless required by applicable law or agreed to in writing, software\n",
"# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
"# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
"# See the License for the specific language governing permissions and\n",
"# limitations under the License."
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "9a05a17e4e2f"
},
"source": [
"# Deploy ADK Agents to Vertex AI Agent Engine Using Inline Source\n",
"\n",
"<table><tbody><tr>\n",
" <td style=\"text-align: center\">\n",
" <a href=\"https://console.cloud.google.com/vertex-ai/colab/import/https:%2F%2Fraw.githubusercontent.com%2FGoogleCloudPlatform%2Fvertex-ai-samples%2Fmain%2Fnotebooks%2Fcommunity%2Fagent_engine%2Fadk_inline_source_deployment.ipynb\">\n",
" <img alt=\"Google Cloud Colab Enterprise logo\" src=\"https://lh3.googleusercontent.com/JmcxdQi-qOpctIvWKgPtrzZdJJK-J3sWE1RsfjZNwshCFgE_9fULcNpuXYTilIR2hjwN\" width=\"32px\"><br> Run in Colab Enterprise\n",
" </a>\n",
" </td>\n",
" <td style=\"text-align: center\">\n",
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/agent_engine/adk_inline_source_deployment.ipynb\">\n",
" <img alt=\"GitHub logo\" src=\"https://github.githubassets.com/assets/GitHub-Mark-ea2971cee799.png\" width=\"32px\"><br> View on GitHub\n",
" </a>\n",
" </td>\n",
"</tr></tbody></table>"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "e6e9014cd53a"
},
"source": [
"## Overview\n",
"\n",
"This notebook demonstrates how to deploy [Google Agent Development Kit (ADK)](https://google.github.io/adk-docs/) agents to [Vertex AI Agent Engine](https://cloud.google.com/vertex-ai/generative-ai/docs/agent-engine/overview) using **inline source deployment**.\n",
"\n",
"I built this pattern while developing a multi-agent system with ADK. The approach is also featured in Google's [Agent Starter Pack community showcase](https://github.com/GoogleCloudPlatform/agent-starter-pack#community-showcase).\n",
"\n",
"Inline source deployment sends your agent's Python source code directly to Agent Engine, avoiding serialization issues that can occur with the default pickle-based deployment.\n",
"\n",
"### Why Inline Source Deployment?\n",
"\n",
"| Approach | How it works | Best for |\n",
"|----------|--------------|----------|\n",
"| Default (pickle) | Serializes agent object | Simple agents, quick prototypes |\n",
"| **Inline source** | Sends Python source directly | Complex agents, custom dependencies |\n",
"\n",
"Inline source deployment is useful when:\n",
"- Your agent uses custom classes that don't serialize well\n",
"- You need precise control over the deployment environment\n",
"- You want to avoid pickle compatibility issues across Python versions\n",
"\n",
"### Objectives\n",
"\n",
"- Create an ADK agent with tools\n",
"- Deploy to Agent Engine using inline source\n",
"- Test the deployed agent\n",
"- Clean up resources\n",
"\n",
"### Costs\n",
"\n",
"This tutorial uses billable components of Google Cloud:\n",
"\n",
"* Vertex AI Agent Engine\n",
"* Vertex AI (Gemini API)\n",
"\n",
"Learn about [Vertex AI pricing](https://cloud.google.com/vertex-ai/pricing) and use the [Pricing Calculator](https://cloud.google.com/products/calculator/) to generate a cost estimate based on your projected usage."
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "4d998a5140b2"
},
"source": [
"## Get started"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "c832028166e3"
},
"source": [
"### Install required packages"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "6b2e64d59bf4"
},
"outputs": [],
"source": [
"# Define requirements once for reuse\n",
"REQUIREMENTS = [\n",
" \"google-cloud-aiplatform[adk,agent_engines]>=1.88.0\",\n",
" \"google-adk>=1.18.0\",\n",
"]\n",
"\n",
"!pip install --upgrade --quiet {\" \".join(REQUIREMENTS)}"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "2f2e1483e7a8"
},
"source": [
"### Restart runtime\n",
"\n",
"After installing packages, restart the runtime to ensure the new packages are loaded."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "d76628df8180"
},
"outputs": [],
"source": [
"import IPython\n",
"\n",
"app = IPython.Application.instance()\n",
"app.kernel.do_shutdown(True)"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "586675a64f57"
},
"source": [
"### Authenticate your notebook environment (Colab only)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "2082bbcb771c"
},
"outputs": [],
"source": [
"import sys\n",
"\n",
"if \"google.colab\" in sys.modules:\n",
" from google.colab import auth\n",
"\n",
" auth.authenticate_user()"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "296a2345bc1a"
},
"source": [
"### Set Google Cloud project information"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "abafb08d10b1"
},
"outputs": [],
"source": [
"import os\n",
"\n",
"PROJECT_ID = \"[your-project-id]\" # @param {type: \"string\", placeholder: \"[your-project-id]\", isTemplate: true}\n",
"if not PROJECT_ID or PROJECT_ID == \"[your-project-id]\":\n",
" PROJECT_ID = str(os.environ.get(\"GOOGLE_CLOUD_PROJECT\"))\n",
"\n",
"LOCATION = \"us-central1\" # @param {type:\"string\"}\n",
"\n",
"STAGING_BUCKET = f\"gs://{PROJECT_ID}-agent-engine-staging\" # @param {type:\"string\"}"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "c7863dcf6a38"
},
"source": [
"### Create staging bucket\n",
"\n",
"Agent Engine requires a staging bucket to store deployment artifacts."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "17ece3f9fb6e"
},
"outputs": [],
"source": [
"!gsutil mb -l {LOCATION} -p {PROJECT_ID} {STAGING_BUCKET} 2>/dev/null || echo \"Bucket already exists\""
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "1029f595561d"
},
"source": [
"### Initialize Vertex AI"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "cac3c1b09203"
},
"outputs": [],
"source": [
"import vertexai\n",
"\n",
"vertexai.init(\n",
" project=PROJECT_ID,\n",
" location=LOCATION,\n",
" staging_bucket=STAGING_BUCKET,\n",
")"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "46fb887b5ea3"
},
"source": [
"## Create an ADK Agent\n",
"\n",
"First, let's create a simple ADK agent with a tool. This agent can answer questions and get the current time."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "9d0dd0808c69"
},
"outputs": [],
"source": [
"import datetime\n",
"from zoneinfo import ZoneInfo, ZoneInfoNotFoundError\n",
"\n",
"from google.adk.agents import LlmAgent\n",
"\n",
"\n",
"def get_current_time(timezone: str = \"UTC\") -> str:\n",
" \"\"\"Get the current time in the specified timezone.\n",
"\n",
" Args:\n",
" timezone: IANA timezone name (e.g., 'America/New_York', 'UTC')\n",
"\n",
" Returns:\n",
" Current time as a formatted string\n",
" \"\"\"\n",
" try:\n",
" tz = ZoneInfo(timezone)\n",
" now = datetime.datetime.now(tz)\n",
" return (\n",
" f\"The current time in {timezone} is {now.strftime('%Y-%m-%d %H:%M:%S %Z')}\"\n",
" )\n",
" except ZoneInfoNotFoundError:\n",
" return f\"Error: Timezone '{timezone}' not found. Please use a valid IANA timezone name (e.g., 'America/New_York', 'UTC').\"\n",
" except Exception as e:\n",
" return f\"An unexpected error occurred while getting time for timezone {timezone}: {e}\"\n",
"\n",
"\n",
"# Create the agent\n",
"agent = LlmAgent(\n",
" name=\"time_agent\",\n",
" model=\"gemini-1.5-flash-001\",\n",
" instruction=\"You are a helpful assistant. Use the get_current_time tool when asked about the time.\",\n",
" tools=[get_current_time],\n",
")\n",
"\n",
"print(f\"Created agent: {agent.name}\")"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "ad0b9cc230ca"
},
"source": [
"### Test the agent locally\n",
"\n",
"Before deploying, let's test the agent locally to make sure it works."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "1d3d71bde31e"
},
"outputs": [],
"source": [
"from google.adk import Runner\n",
"from google.adk.sessions import InMemorySessionService\n",
"from google.genai import types\n",
"\n",
"# Create a local runner\n",
"runner = Runner(\n",
" agent=agent,\n",
" app_name=\"test_app\",\n",
" session_service=InMemorySessionService(),\n",
")\n",
"\n",
"# Create a session and test\n",
"session = runner.session_service.create_session(\n",
" app_name=\"test_app\",\n",
" user_id=\"test_user\",\n",
")\n",
"\n",
"# Send a test message\n",
"response = runner.run(\n",
" user_id=\"test_user\",\n",
" session_id=session.id,\n",
" new_message=types.Content(\n",
" role=\"user\",\n",
" parts=[types.Part(text=\"What time is it in New York?\")],\n",
" ),\n",
")\n",
"\n",
"# Print the response\n",
"for event in response:\n",
" if hasattr(event, \"content\") and event.content:\n",
" for part in event.content.parts:\n",
" if hasattr(part, \"text\") and part.text:\n",
" print(part.text)"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "06bdc3c10ea6"
},
"source": [
"## Deploy Using Inline Source\n",
"\n",
"Now let's deploy the agent to Agent Engine using inline source deployment.\n",
"\n",
"The key difference from default deployment is using `AgentEngineApp.from_agent()` which packages the source code directly."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "d641a5e29291"
},
"outputs": [],
"source": [
"from vertexai import agent_engines\n",
"from vertexai.agent_engines import AgentEngineApp\n",
"\n",
"# Create the app from the agent (inline source)\n",
"app = AgentEngineApp.from_agent(\n",
" agent=agent,\n",
" enable_tracing=True,\n",
")\n",
"\n",
"print(\"App created successfully\")"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "cfd8fddbf181"
},
"source": [
"### Deploy to Agent Engine"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "ebd6642375a8"
},
"outputs": [],
"source": [
"# Deploy the app\n",
"# This may take a few minutes\n",
"remote_app = agent_engines.create(\n",
" app,\n",
" requirements=REQUIREMENTS,\n",
" display_name=\"time-agent-inline\",\n",
")\n",
"\n",
"print(f\"Deployed agent: {remote_app.resource_name}\")"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "43b739d2f980"
},
"source": [
"## Test the Deployed Agent\n",
"\n",
"Now let's test the deployed agent by sending it a query."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "8ec0598a1138"
},
"outputs": [],
"source": [
"# Create a session with the remote agent\n",
"remote_session = remote_app.create_session(user_id=\"test_user\")\n",
"\n",
"print(f\"Created session: {remote_session['id']}\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "f85603f0e4fd"
},
"outputs": [],
"source": [
"# Query the agent\n",
"response = remote_app.stream_query(\n",
" user_id=\"test_user\",\n",
" session_id=remote_session[\"id\"],\n",
" message=\"What time is it in Tokyo?\",\n",
")\n",
"\n",
"# Print the response\n",
"for chunk in response:\n",
" if hasattr(chunk, \"content\") and chunk.content:\n",
" for part in chunk.content.parts:\n",
" if hasattr(part, \"text\") and part.text:\n",
" print(part.text, end=\"\")\n",
"\n",
"print() # Newline at end"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "e70d97acf3a3"
},
"source": [
"## Cleaning up\n",
"\n",
"Delete the deployed agent to avoid ongoing charges."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "6d2d34a6ba7c"
},
"outputs": [],
"source": [
"# Delete the deployed agent\n",
"delete_agent = False # @param {type:\"boolean\"}\n",
"\n",
"if delete_agent:\n",
" remote_app.delete()\n",
" print(\"Agent deleted successfully\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "35870ecf063a"
},
"outputs": [],
"source": [
"# Delete the staging bucket\n",
"delete_bucket = False # @param {type:\"boolean\"}\n",
"\n",
"if delete_bucket:\n",
" !gsutil rm -r {STAGING_BUCKET}\n",
" print(\"Bucket deleted successfully\")"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "11f2cc08759a"
},
"source": [
"## Summary\n",
"\n",
"In this notebook, you learned how to:\n",
"\n",
"1. Create an ADK agent with tools\n",
"2. Test the agent locally\n",
"3. Deploy to Agent Engine using inline source with `AgentEngineApp.from_agent()`\n",
"4. Query the deployed agent\n",
"\n",
"### Next steps\n",
"\n",
"- Learn more about [ADK](https://google.github.io/adk-docs/)\n",
"- Explore [Agent Engine documentation](https://cloud.google.com/vertex-ai/generative-ai/docs/agent-engine/overview)\n",
"- Try adding more complex tools and multi-agent orchestration"
]
}
],
"metadata": {
"colab": {
"name": "adk_inline_source_deployment.ipynb",
"toc_visible": true
},
"kernelspec": {
"display_name": "Python 3",
"name": "python3"
}
},
"nbformat": 4,
"nbformat_minor": 0
}
-93
View File
@@ -1,93 +0,0 @@
![AlphaGenome header image](https://raw.githubusercontent.com/google-deepmind/alphagenome/refs/heads/main/docs/source/_static/header.png)
# AlphaGenome
[**Overview**](#overview) | [**Use Cases**](#use-cases) | [**Documentation**](#documentation) | [**Pricing**](#pricing) | [**Quick start**](#quick-start)
## Overview
**Disclaimer:** *Experimental*.
*The AlphaGenome Private Preview is a "Pre-GA Offering" subject to the "Pre-GA
Offerings Terms" in the General Service Terms section of the Google Cloud
[Service Specific Terms](https://cloud.google.com/terms/service-terms). It is
also a “Generative AI Preview Product” as defined in and subject to the
[Additional Terms for Generative AI Preview Products](https://cloud.google.com/trustedtester/aitos?e=48754805&hl=en).
Pre-GA products are available "as is" and might have limited support. For more
information, see the [launch stage](https://cloud.google.com/products?e=48754805#product-launch-stages)
descriptions.*
Access to the AlphaGenome model capabilities requires application and approval.
Users must be added to an allowlist to use the service.
If you are interested in applying to the program, **Request Access** above.
&nbsp;
AlphaGenome is Google DeepMind’s unifying model for deciphering the regulatory
code within DNA sequences.
AlphaGenome offers multimodal predictions, encompassing diverse functional
outputs such as gene expression, splicing patterns, chromatin features, and
contact maps (see diagram below). The model analyzes DNA sequences of up to 1
million base pairs in length and can deliver predictions at single base-pair
resolution for most outputs.
Training data was sourced from large public consortia including
[ENCODE](http://encodeproject.org/), [GTEx](https://www.gtexportal.org/),
[4D Nucleome](https://4dnucleome.org/) and
[FANTOM5](https://fantom.gsc.riken.jp/5/), which experimentally measured these
properties covering important modalities of gene regulation across hundreds of
human and mouse cell types and tissues.
![Diagram showing an overview of the AlphaGenome model architecture and its inputs/outputs](https://www.alphagenomedocs.com/_images/model_overview.png)
## Use Cases
* **Sequence-to-function predictions:** Predict multiple functional tracks (such as gene expression, splicing) from DNA sequences across a wide variety of tissues and cell types.
* **Variant effect scoring:** Assess the impact of genetic variants by comparing predictions for the reference and alternative alleles and summarising the differences between them.
* **Identify functional regions:** Use in silico mutagenesis (ISM) to identify functionally important regions in the DNA sequence.
* **Human and mouse capability:** Generate predictions for both human and mouse genomes.
## Documentation
This API provides access to AlphaGenome, Google DeepMind's unifying model for
deciphering the regulatory code within DNA sequences. AlphaGenome offers
multimodal predictions, encompassing diverse functional outputs including gene
expression, splicing patterns, chromatin features, and contact maps (see diagram
below). The model analyzes up to 1 million base pairs of DNA sequence and can
deliver predictions at single base-pair resolution for most modalities.
AlphaGenome achieves state-of-the-art performance across a range of genomic
prediction benchmarks, including diverse variant effect prediction tasks.
The Google Cloud API for AlphaGenome provides a way for Google Cloud customers
to explore the AlphaGenome API for commercial use cases. This API is in private
preview (Request Access above). Once allowlisted, customers can access the API
directly or use the [colab](cloudai_alphagenome_vai_quickstart.ipynb).
### Acknowledgements
*Avsec, Ž., Latysheva, N., Cheng, J., Novati, G., Taylor, K. R., Ward, T., ... Kohli, P. (2025). AlphaGenome: advancing regulatory variant effect prediction with a unified DNA sequence model. bioRxiv.* [https://doi.org/10.1101/2025.06.25.661532](https://doi.org/10.1101/2025.06.25.661532)
### Contact
If you have any questions on using these models on Google Cloud please contact:
[alphagenome-cloud-external@google.com](mailto:alphagenome-cloud-external@google.com) or join the community [Discourse](https://www.alphagenomecommunity.com/) for more generic questions on AlphaGenome.
### Links
* Read our [paper](https://doi.org/10.1101/2025.06.25.661532)
* Read our [blog post](https://deepmind.google/discover/blog/alphagenome-ai-for-better-understanding-the-genome)
* Join the [community](https://www.alphagenomecommunity.com/)
* Check out the [AlphaGenome 101 Video](https://youtu.be/Xbvloe13nak)
## Pricing
Access to AlphaGenome on Vertex AI is currently restricted.
To utilize these models via this service:
* You must **Request Access** using your Google contact.
* Your application will be reviewed, and if approved, you will be **added to
an allowlist**.
* Only allowlisted users can access the API
* **Pricing information** will be shared directly with users upon approval
and placement on the allowlist.
## Quick start
The quickest way to get started with the AlphaGenome in Google Cloud Platform is to run [our example notebook](cloudai_alphagenome_vai_quickstart.ipynb) in [Google Colab](https://colab.research.google.com/).
File diff suppressed because one or more lines are too long
@@ -42,7 +42,7 @@
" </td>\n",
" <td>\n",
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/bigquery_ml/Anomaly_detection_in_Cloud_Audit_logs_with_BQML.ipynb\">\n",
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
" View on GitHub\n",
" </a>\n",
" </td>\n",
@@ -52,7 +52,7 @@
" </td>\n",
" <td>\n",
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/bigquery_ml/bq_ml_with_vision_translation_nlp.ipynb\">\n",
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
" View on GitHub\n",
" </a>\n",
" </td>\n",
@@ -40,7 +40,7 @@
" </td>\n",
" <td>\n",
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/bigquery_ml/bqml-online-prediction.ipynb\">\n",
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
" View on GitHub\n",
" </a>\n",
" </td>\n",
@@ -359,7 +359,7 @@
},
"outputs": [],
"source": [
"! gcloud storage buckets create --location={REGION} --project={PROJECT_ID} {BUCKET_URI}"
"! gsutil mb -l {REGION} -p {PROJECT_ID} {BUCKET_URI}"
]
},
{
@@ -1098,7 +1098,7 @@
" ! bq rm -r -f $PROJECT_ID:$BQ_DATASET_NAME\n",
"# delete the Cloud Storage bucket\n",
"if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
" ! gcloud storage rm --recursive $BUCKET_URI"
" ! gsutil -m rm -r $BUCKET_URI"
]
}
],
@@ -38,7 +38,7 @@
" </td>\n",
" <td>\n",
" <a href=\"github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/cohere/cohere_embedding_with_matching_engine.ipynb\">\n",
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
" View on GitHub\n",
" </a>\n",
" </td>\n",
@@ -449,7 +449,7 @@
},
"outputs": [],
"source": [
"! gcloud storage buckets create --location=$REGION $BUCKET_NAME"
"! gsutil mb -l $REGION $BUCKET_NAME"
]
},
{
@@ -470,7 +470,7 @@
"outputs": [],
"source": [
"# this will not return anything if the bucket is empty\n",
"! gcloud storage ls --all-versions --long $BUCKET_NAME"
"! gsutil ls -al $BUCKET_NAME"
]
},
{
@@ -588,7 +588,7 @@
"# NOTE: Everything in this GCS DIR will be DELETED before uploading the data.\n",
"# A CommandException is expected if no data is present\n",
"\n",
"! gcloud storage rm --recursive --continue-on-error {BUCKET_NAME}/*"
"! gsutil rm -rf {BUCKET_NAME}/*"
]
},
{
@@ -599,7 +599,7 @@
},
"outputs": [],
"source": [
"! gcloud storage cp cohere_embeddings.json {BUCKET_NAME}/cohere_embeddings.json"
"! gsutil cp cohere_embeddings.json {BUCKET_NAME}/cohere_embeddings.json"
]
},
{
@@ -610,7 +610,7 @@
},
"outputs": [],
"source": [
"! gcloud storage ls {BUCKET_NAME}"
"! gsutil ls {BUCKET_NAME}"
]
},
{
@@ -41,7 +41,7 @@
" </td>\n",
" <td>\n",
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/vertex_ai_experiments_classification.ipynb\">\n",
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
" View on GitHub\n",
" </a>\n",
" </td>\n",
@@ -537,7 +537,7 @@
},
"outputs": [],
"source": [
"! gcloud storage buckets create --location=$REGION --project=$PROJECT_ID $BUCKET_URI"
"! gsutil mb -l $REGION -p $PROJECT_ID $BUCKET_URI"
]
},
{
@@ -557,7 +557,7 @@
},
"outputs": [],
"source": [
"! gcloud storage ls --all-versions --long $BUCKET_URI"
"! gsutil ls -al $BUCKET_URI"
]
},
{
@@ -627,9 +627,9 @@
},
"outputs": [],
"source": [
"! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n",
"! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectCreator $BUCKET_URI\n",
"\n",
"! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer"
"! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectViewer $BUCKET_URI"
]
},
{
@@ -1736,7 +1736,7 @@
"delete_bucket = False\n",
"\n",
"if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
" ! gcloud storage rm --recursive --continue-on-error {BUCKET_URI}"
" ! gsutil rm -rf {BUCKET_URI}"
]
}
],
@@ -38,7 +38,7 @@
" </td>\n",
" <td>\n",
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/explainable_ai/SDK_Custom_Container_XAI.ipynb\">\n",
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
" View on GitHub\n",
" </a>\n",
" </td>\n",
@@ -405,7 +405,7 @@
},
"outputs": [],
"source": [
"! gcloud storage buckets create --location=$REGION --project=$PROJECT_ID $BUCKET_URI"
"! gsutil mb -l $REGION -p $PROJECT_ID $BUCKET_URI"
]
},
{
@@ -425,7 +425,7 @@
},
"outputs": [],
"source": [
"! gcloud storage ls --all-versions --long $BUCKET_URI"
"! gsutil ls -al $BUCKET_URI"
]
},
{
@@ -698,7 +698,7 @@
},
"outputs": [],
"source": [
"!gcloud storage cp app/model.joblib {BUCKET_URI}/{MODEL_ARTIFACT_DIR}/"
"!gsutil cp app/model.joblib {BUCKET_URI}/{MODEL_ARTIFACT_DIR}/"
]
},
{
@@ -1516,7 +1516,7 @@
"\n",
"delete_bucket = False\n",
"if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
" ! gcloud storage rm --recursive --continue-on-error {BUCKET_URI}"
" ! gsutil rm -rf {BUCKET_URI}"
]
}
],
@@ -637,20 +637,20 @@
},
"outputs": [],
"source": [
"# Upload CSV data to Cloud Storage by passing gcloud storage commands to system\n",
"# Upload CSV data to Cloud Storage by passing gsutil commands to system\n",
"gcs_url <- paste0(\"gs://\", BUCKET_NAME, \"/\")\n",
"\n",
"command <- paste(\"gcloud storage buckets create\", gcs_url)\n",
"command <- paste(\"gsutil mb\", gcs_url)\n",
"\n",
"system(command)\n",
"\n",
"gcs_data_dir <- paste0(\"gs://\", BUCKET_NAME, \"/data\")\n",
"\n",
"command <- paste(\"gcloud storage cp data/*_data.csv\", gcs_data_dir)\n",
"command <- paste(\"gsutil cp data/*_data.csv\", gcs_data_dir)\n",
"\n",
"system(command)\n",
"\n",
"command <- paste(\"gcloud storage ls --long\", gcs_data_dir)\n",
"command <- paste(\"gsutil ls -l\", gcs_data_dir)\n",
"\n",
"system(command, intern = TRUE)"
]
@@ -679,4 +679,4 @@
},
"nbformat": 4,
"nbformat_minor": 4
}
}
@@ -38,7 +38,7 @@
" </td>\n",
" <td>\n",
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/official/feature_store/gapic-feature-store.ipynb\">\n",
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
" View on GitHub\n",
" </a>\n",
" </td>\n",
@@ -34,7 +34,7 @@
"<table align=\"left\">\n",
" <td>\n",
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage6/get_started_vertex_feature_store_serving.ipynb\">\n",
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
" View on GitHub\n",
" </a>\n",
" </td>\n",
@@ -471,7 +471,8 @@
},
"outputs": [],
"source": [
"! gcloud storage buckets create --location=$REGION $BUCKET_URI" ]
"! gsutil mb -l $REGION $BUCKET_URI"
]
},
{
"cell_type": "markdown",
@@ -490,7 +491,8 @@
},
"outputs": [],
"source": [
"! gcloud storage ls --all-versions --long $BUCKET_URI" ]
"! gsutil ls -al $BUCKET_URI"
]
},
{
"cell_type": "markdown",
@@ -509,7 +511,8 @@
},
"outputs": [],
"source": [
"! gcloud storage buckets update --uniform-bucket-level-access {BUCKET_URI}" ]
"! gsutil uniformbucketlevelaccess set on {BUCKET_URI}"
]
},
{
"cell_type": "markdown",
@@ -1907,7 +1910,9 @@
},
"outputs": [],
"source": [
"! gcloud storage rm --recursive $BUCKET_URI\n", "! gcloud storage buckets delete $BUCKET_URI" ]
"! gsutil -m rm -r $BUCKET_URI\n",
"! gsutil rb $BUCKET_URI"
]
}
],
"metadata": {
@@ -38,7 +38,7 @@
" </td>\n",
" <td>\n",
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/feature_store/mobile_gaming/mobile_gaming_feature_store.ipynb\">\n",
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
" View on GitHub\n",
" </a>\n",
" </td>\n",
@@ -37,7 +37,7 @@
" </td>\n",
" <td>\n",
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/feature_store/sdk-feature-store.ipynb\">\n",
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
" View on GitHub\n",
" </a>\n",
" </td>\n",
@@ -39,7 +39,7 @@
" </td>\n",
" <td>\n",
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/gapic/automl/showcase_automl_image_classification_batch.ipynb\">\n",
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
" View on GitHub\n",
" </a>\n",
" </td>\n",
@@ -421,7 +421,7 @@
},
"outputs": [],
"source": [
"! gcloud storage buckets create --location=$REGION $BUCKET_NAME"
"! gsutil mb -l $REGION $BUCKET_NAME"
]
},
{
@@ -441,7 +441,7 @@
},
"outputs": [],
"source": [
"! gcloud storage ls --all-versions --long $BUCKET_NAME"
"! gsutil ls -al $BUCKET_NAME"
]
},
{
@@ -478,7 +478,9 @@
"import time\n",
"\n",
"from google.cloud.aiplatform import gapic as aip\n",
"from google.protobuf import json_format"
"from google.protobuf import json_format\n",
"from google.protobuf.json_format import MessageToJson, ParseDict\n",
"from google.protobuf.struct_pb2 import Struct, Value"
]
},
{
@@ -884,11 +886,11 @@
"else:\n",
" FILE = IMPORT_FILE\n",
"\n",
"count = ! gcloud storage cat $FILE | wc -l\n",
"count = ! gsutil cat $FILE | wc -l\n",
"print(\"Number of Examples\", int(count[0]))\n",
"\n",
"print(\"First 10 rows\")\n",
"! gcloud storage cat $FILE | head"
"! gsutil cat $FILE | head"
]
},
{
@@ -1327,7 +1329,7 @@
},
"outputs": [],
"source": [
"test_items = !gcloud storage cat $IMPORT_FILE | head -n2\n",
"test_items = !gsutil cat $IMPORT_FILE | head -n2\n",
"if len(str(test_items[0]).split(\",\")) == 3:\n",
" _, test_item_1, test_label_1 = str(test_items[0]).split(\",\")\n",
" _, test_item_2, test_label_2 = str(test_items[1]).split(\",\")\n",
@@ -1361,8 +1363,8 @@
"file_1 = test_item_1.split(\"/\")[-1]\n",
"file_2 = test_item_2.split(\"/\")[-1]\n",
"\n",
"! gcloud storage cp $test_item_1 $BUCKET_NAME/$file_1\n",
"! gcloud storage cp $test_item_2 $BUCKET_NAME/$file_2\n",
"! gsutil cp $test_item_1 $BUCKET_NAME/$file_1\n",
"! gsutil cp $test_item_2 $BUCKET_NAME/$file_2\n",
"\n",
"test_item_1 = BUCKET_NAME + \"/\" + file_1\n",
"test_item_2 = BUCKET_NAME + \"/\" + file_2"
@@ -1406,7 +1408,7 @@
" f.write(json.dumps(data) + \"\\n\")\n",
"\n",
"print(gcs_input_uri)\n",
"! gcloud storage cat $gcs_input_uri"
"! gsutil cat $gcs_input_uri"
]
},
{
@@ -1690,8 +1692,8 @@
"outputs": [],
"source": [
"def get_latest_predictions(gcs_out_dir):\n",
" \"\"\"Get the latest prediction subfolder using the timestamp in the subfolder name\"\"\"\n",
" folders = !gcloud storage ls $gcs_out_dir\n",
" \"\"\" Get the latest prediction subfolder using the timestamp in the subfolder name\"\"\"\n",
" folders = !gsutil ls $gcs_out_dir\n",
" latest = \"\"\n",
" for folder in folders:\n",
" subfolder = folder.split(\"/\")[-2]\n",
@@ -1709,9 +1711,9 @@
" raise Exception(\"Batch Job Failed\")\n",
" else:\n",
" folder = get_latest_predictions(predictions)\n",
" ! gcloud storage ls $folder/prediction*.jsonl\n",
" ! gsutil ls $folder/prediction*.jsonl\n",
"\n",
" ! gcloud storage cat $folder/prediction*.jsonl\n",
" ! gsutil cat $folder/prediction*.jsonl\n",
" break\n",
" time.sleep(60)"
]
@@ -1806,7 +1808,7 @@
" print(e)\n",
"\n",
"if delete_bucket and \"BUCKET_NAME\" in globals():\n",
" ! gcloud storage rm --recursive $BUCKET_NAME"
" ! gsutil rm -r $BUCKET_NAME"
]
}
],
@@ -39,7 +39,7 @@
" </td>\n",
" <td>\n",
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/gapic/automl/showcase_automl_image_classification_export_edge.ipynb\">\n",
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
" View on GitHub\n",
" </a>\n",
" </td>\n",
@@ -416,7 +416,7 @@
},
"outputs": [],
"source": [
"! gcloud storage buckets create --location=$REGION $BUCKET_NAME"
"! gsutil mb -l $REGION $BUCKET_NAME"
]
},
{
@@ -436,7 +436,7 @@
},
"outputs": [],
"source": [
"! gcloud storage ls --all-versions --long $BUCKET_NAME"
"! gsutil ls -al $BUCKET_NAME"
]
},
{
@@ -474,6 +474,7 @@
"\n",
"from google.cloud.aiplatform import gapic as aip\n",
"from google.protobuf import json_format\n",
"from google.protobuf.json_format import MessageToJson, ParseDict\n",
"from google.protobuf.struct_pb2 import Struct, Value"
]
},
@@ -783,11 +784,11 @@
"else:\n",
" FILE = IMPORT_FILE\n",
"\n",
"count = ! gcloud storage cat $FILE | wc -l\n",
"count = ! gsutil cat $FILE | wc -l\n",
"print(\"Number of Examples\", int(count[0]))\n",
"\n",
"print(\"First 10 rows\")\n",
"! gcloud storage cat $FILE | head"
"! gsutil cat $FILE | head"
]
},
{
@@ -1214,6 +1215,7 @@
" }\n",
" response = clients[\"model\"].export_model(name=name, output_config=output_config)\n",
" print(\"Long running operation:\", response.operation.name)\n",
" result = response.result(timeout=1800)\n",
" metadata = response.operation.metadata\n",
" artifact_uri = str(metadata.value).split(\"\\\\\")[-1][4:-1]\n",
" print(\"Artifact Uri\", artifact_uri)\n",
@@ -1242,9 +1244,9 @@
},
"outputs": [],
"source": [
"! gcloud storage ls $model_package\n",
"! gsutil ls $model_package\n",
"# Download the model artifacts\n",
"! gcloud storage cp --recursive $model_package tflite\n",
"! gsutil cp -r $model_package tflite\n",
"\n",
"tflite_path = \"tflite/model.tflite\""
]
@@ -1303,7 +1305,7 @@
},
"outputs": [],
"source": [
"test_items = ! gcloud storage cat $IMPORT_FILE | head -n1\n",
"test_items = ! gsutil cat $IMPORT_FILE | head -n1\n",
"test_item = test_items[0].split(\",\")[0]\n",
"\n",
"with tf.io.gfile.GFile(test_item, \"rb\") as f:\n",
@@ -1447,7 +1449,7 @@
" print(e)\n",
"\n",
"if delete_bucket and \"BUCKET_NAME\" in globals():\n",
" ! gcloud storage rm --recursive $BUCKET_NAME"
" ! gsutil rm -r $BUCKET_NAME"
]
}
],
@@ -39,7 +39,7 @@
" </td>\n",
" <td>\n",
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/gapic/automl/showcase_automl_image_classification_online.ipynb\">\n",
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
" View on GitHub\n",
" </a>\n",
" </td>\n",
@@ -721,10 +721,12 @@
"else:\n",
" FILE = IMPORT_FILE\n",
"\n",
"count = ! gcloud storage cat $FILE | wc -l\n", "print(\"Number of Examples\", int(count[0]))\n",
"count = ! gsutil cat $FILE | wc -l\n",
"print(\"Number of Examples\", int(count[0]))\n",
"\n",
"print(\"First 10 rows\")\n",
"! gcloud storage cat $FILE | head" ]
"! gsutil cat $FILE | head"
]
},
{
"cell_type": "markdown",
@@ -1340,7 +1342,8 @@
},
"outputs": [],
"source": [
"test_item = !gcloud storage cat $IMPORT_FILE | head -n1\n", "if len(str(test_item[0]).split(\",\")) == 3:\n",
"test_item = !gsutil cat $IMPORT_FILE | head -n1\n",
"if len(str(test_item[0]).split(\",\")) == 3:\n",
" _, test_item, test_label = str(test_item[0]).split(\",\")\n",
"else:\n",
" test_item, test_label = str(test_item[0]).split(\",\")\n",
@@ -1565,7 +1568,8 @@
" print(e)\n",
"\n",
"if delete_bucket and \"BUCKET_NAME\" in globals():\n",
" ! gcloud storage rm --recursive $BUCKET_NAME" ]
" ! gsutil rm -r $BUCKET_NAME"
]
}
],
"metadata": {
@@ -39,7 +39,7 @@
" </td>\n",
" <td>\n",
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/gapic/automl/showcase_automl_image_classification_online_proxy.ipynb\">\n",
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
" View on GitHub\n",
" </a>\n",
" </td>\n",
@@ -721,10 +721,12 @@
"else:\n",
" FILE = IMPORT_FILE\n",
"\n",
"count = ! gcloud storage cat $FILE | wc -l\n", "print(\"Number of Examples\", int(count[0]))\n",
"count = ! gsutil cat $FILE | wc -l\n",
"print(\"Number of Examples\", int(count[0]))\n",
"\n",
"print(\"First 10 rows\")\n",
"! gcloud storage cat $FILE | head" ]
"! gsutil cat $FILE | head"
]
},
{
"cell_type": "markdown",
@@ -1340,7 +1342,8 @@
},
"outputs": [],
"source": [
"test_item = !gcloud storage cat $IMPORT_FILE | head -n1\n", "if len(str(test_item[0]).split(\",\")) == 3:\n",
"test_item = !gsutil cat $IMPORT_FILE | head -n1\n",
"if len(str(test_item[0]).split(\",\")) == 3:\n",
" _, test_item, test_label = str(test_item[0]).split(\",\")\n",
"else:\n",
" test_item, test_label = str(test_item[0]).split(\",\")\n",
@@ -1744,7 +1747,8 @@
" print(e)\n",
"\n",
"if delete_bucket and \"BUCKET_NAME\" in globals():\n",
" ! gcloud storage rm --recursive $BUCKET_NAME" ]
" ! gsutil rm -r $BUCKET_NAME"
]
}
],
"metadata": {
@@ -39,7 +39,7 @@
" </td>\n",
" <td>\n",
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/gapic/automl/showcase_automl_image_object_detection_batch.ipynb\">\n",
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
" View on GitHub\n",
" </a>\n",
" </td>\n",
@@ -421,7 +421,7 @@
},
"outputs": [],
"source": [
"! gcloud storage buckets create --location=$REGION $BUCKET_NAME"
"! gsutil mb -l $REGION $BUCKET_NAME"
]
},
{
@@ -441,7 +441,7 @@
},
"outputs": [],
"source": [
"! gcloud storage ls --all-versions --long $BUCKET_NAME"
"! gsutil ls -al $BUCKET_NAME"
]
},
{
@@ -478,7 +478,9 @@
"import time\n",
"\n",
"from google.cloud.aiplatform import gapic as aip\n",
"from google.protobuf import json_format"
"from google.protobuf import json_format\n",
"from google.protobuf.json_format import MessageToJson, ParseDict\n",
"from google.protobuf.struct_pb2 import Struct, Value"
]
},
{
@@ -885,11 +887,11 @@
"else:\n",
" FILE = IMPORT_FILE\n",
"\n",
"count = ! gcloud storage cat $FILE | wc -l\n",
"count = ! gsutil cat $FILE | wc -l\n",
"print(\"Number of Examples\", int(count[0]))\n",
"\n",
"print(\"First 10 rows\")\n",
"! gcloud storage cat $FILE | head"
"! gsutil cat $FILE | head"
]
},
{
@@ -1331,7 +1333,7 @@
},
"outputs": [],
"source": [
"test_items = !gcloud storage cat $IMPORT_FILE | head -n2\n",
"test_items = !gsutil cat $IMPORT_FILE | head -n2\n",
"cols_1 = str(test_items[0]).split(\",\")\n",
"cols_2 = str(test_items[1]).split(\",\")\n",
"if len(cols_1) == 11:\n",
@@ -1371,8 +1373,8 @@
"file_1 = test_item_1.split(\"/\")[-1]\n",
"file_2 = test_item_2.split(\"/\")[-1]\n",
"\n",
"! gcloud storage cp $test_item_1 $BUCKET_NAME/$file_1\n",
"! gcloud storage cp $test_item_2 $BUCKET_NAME/$file_2\n",
"! gsutil cp $test_item_1 $BUCKET_NAME/$file_1\n",
"! gsutil cp $test_item_2 $BUCKET_NAME/$file_2\n",
"\n",
"test_item_1 = BUCKET_NAME + \"/\" + file_1\n",
"test_item_2 = BUCKET_NAME + \"/\" + file_2"
@@ -1416,7 +1418,7 @@
" f.write(json.dumps(data) + \"\\n\")\n",
"\n",
"print(gcs_input_uri)\n",
"! gcloud storage cat $gcs_input_uri"
"! gsutil cat $gcs_input_uri"
]
},
{
@@ -1702,8 +1704,8 @@
"outputs": [],
"source": [
"def get_latest_predictions(gcs_out_dir):\n",
" \"\"\"Get the latest prediction subfolder using the timestamp in the subfolder name\"\"\"\n",
" folders = !gcloud storage ls $gcs_out_dir\n",
" \"\"\" Get the latest prediction subfolder using the timestamp in the subfolder name\"\"\"\n",
" folders = !gsutil ls $gcs_out_dir\n",
" latest = \"\"\n",
" for folder in folders:\n",
" subfolder = folder.split(\"/\")[-2]\n",
@@ -1721,9 +1723,9 @@
" raise Exception(\"Batch Job Failed\")\n",
" else:\n",
" folder = get_latest_predictions(predictions)\n",
" ! gcloud storage ls $folder/prediction*.jsonl\n",
" ! gsutil ls $folder/prediction*.jsonl\n",
"\n",
" ! gcloud storage cat $folder/prediction*.jsonl\n",
" ! gsutil cat $folder/prediction*.jsonl\n",
" break\n",
" time.sleep(60)"
]
@@ -1818,7 +1820,7 @@
" print(e)\n",
"\n",
"if delete_bucket and \"BUCKET_NAME\" in globals():\n",
" ! gcloud storage rm --recursive $BUCKET_NAME"
" ! gsutil rm -r $BUCKET_NAME"
]
}
],
@@ -39,7 +39,7 @@
" </td>\n",
" <td>\n",
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/gapic/automl/showcase_automl_image_object_detection_export_edge.ipynb\">\n",
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
" View on GitHub\n",
" </a>\n",
" </td>\n",
@@ -416,7 +416,7 @@
},
"outputs": [],
"source": [
"! gcloud storage buckets create --location $REGION $BUCKET_NAME"
"! gsutil mb -l $REGION $BUCKET_NAME"
]
},
{
@@ -436,7 +436,7 @@
},
"outputs": [],
"source": [
"! gcloud storage ls --all-versions --long $BUCKET_NAME"
"! gsutil ls -al $BUCKET_NAME"
]
},
{
@@ -473,7 +473,9 @@
"import time\n",
"\n",
"from google.cloud.aiplatform import gapic as aip\n",
"from google.protobuf import json_format\n"
"from google.protobuf import json_format\n",
"from google.protobuf.json_format import MessageToJson, ParseDict\n",
"from google.protobuf.struct_pb2 import Struct, Value"
]
},
{
@@ -783,11 +785,11 @@
"else:\n",
" FILE = IMPORT_FILE\n",
"\n",
"count = ! gcloud storage cat $FILE | wc -l\n",
"count = ! gsutil cat $FILE | wc -l\n",
"print(\"Number of Examples\", int(count[0]))\n",
"\n",
"print(\"First 10 rows\")\n",
"! gcloud storage cat $FILE | head"
"! gsutil cat $FILE | head"
]
},
{
@@ -1216,6 +1218,7 @@
" }\n",
" response = clients[\"model\"].export_model(name=name, output_config=output_config)\n",
" print(\"Long running operation:\", response.operation.name)\n",
" result = response.result(timeout=1800)\n",
" metadata = response.operation.metadata\n",
" artifact_uri = str(metadata.value).split(\"\\\\\")[-1][4:-1]\n",
" print(\"Artifact Uri\", artifact_uri)\n",
@@ -1244,9 +1247,9 @@
},
"outputs": [],
"source": [
"! gcloud storage ls $model_package\n",
"! gsutil ls $model_package\n",
"# Download the model artifacts\n",
"! gcloud storage cp --recursive $model_package tflite\n",
"! gsutil cp -r $model_package tflite\n",
"\n",
"tflite_path = \"tflite/model.tflite\""
]
@@ -1305,7 +1308,7 @@
},
"outputs": [],
"source": [
"test_items = ! gcloud storage cat $IMPORT_FILE | head -n1\n",
"test_items = ! gsutil cat $IMPORT_FILE | head -n1\n",
"test_item = test_items[0].split(\",\")[0]\n",
"\n",
"with tf.io.gfile.GFile(test_item, \"rb\") as f:\n",
@@ -1449,7 +1452,7 @@
" print(e)\n",
"\n",
"if delete_bucket and \"BUCKET_NAME\" in globals():\n",
" ! gcloud storage rm --recursive $BUCKET_NAME"
" ! gsutil rm -r $BUCKET_NAME"
]
}
],
@@ -39,7 +39,7 @@
" </td>\n",
" <td>\n",
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/gapic/automl/showcase_automl_image_object_detection_online.ipynb\">\n",
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
" View on GitHub\n",
" </a>\n",
" </td>\n",
@@ -396,7 +396,9 @@
"import time\n",
"\n",
"from google.cloud.aiplatform import gapic as aip\n",
"from google.protobuf import json_format"
"from google.protobuf import json_format\n",
"from google.protobuf.json_format import MessageToJson, ParseDict\n",
"from google.protobuf.struct_pb2 import Struct, Value"
]
},
{
@@ -720,11 +722,11 @@
"else:\n",
" FILE = IMPORT_FILE\n",
"\n",
"count = ! gcloud storage cat $FILE | wc -l\n",
"count = ! gsutil cat $FILE | wc -l\n",
"print(\"Number of Examples\", int(count[0]))\n",
"\n",
"print(\"First 10 rows\")\n",
"! gcloud storage cat $FILE | head"
"! gsutil cat $FILE | head"
]
},
{
@@ -1343,7 +1345,7 @@
},
"outputs": [],
"source": [
"test_items = !gcloud storage cat $IMPORT_FILE | head -n1\n",
"test_items = !gsutil cat $IMPORT_FILE | head -n1\n",
"cols = str(test_items[0]).split(\",\")\n",
"if len(cols) == 11:\n",
" test_item = str(cols[1])\n",
@@ -1572,7 +1574,7 @@
" print(e)\n",
"\n",
"if delete_bucket and \"BUCKET_NAME\" in globals():\n",
" ! gcloud storage rm --recursive $BUCKET_NAME"
" ! gsutil rm -r $BUCKET_NAME"
]
}
],
@@ -39,7 +39,7 @@
" </td>\n",
" <td>\n",
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/gapic/automl/showcase_automl_image_segmentation_batch.ipynb\">\n",
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
" View on GitHub\n",
" </a>\n",
" </td>\n",
@@ -421,7 +421,8 @@
},
"outputs": [],
"source": [
"! gcloud storage buckets create --location $REGION $BUCKET_NAME" ]
"! gsutil mb -l $REGION $BUCKET_NAME"
]
},
{
"cell_type": "markdown",
@@ -440,7 +441,8 @@
},
"outputs": [],
"source": [
"! gcloud storage ls --all-versions --long $BUCKET_NAME" ]
"! gsutil ls -al $BUCKET_NAME"
]
},
{
"cell_type": "markdown",
@@ -889,10 +891,12 @@
"else:\n",
" FILE = IMPORT_FILE\n",
"\n",
"count = ! gcloud storage cat $FILE | wc -l\n", "print(\"Number of Examples\", int(count[0]))\n",
"count = ! gsutil cat $FILE | wc -l\n",
"print(\"Number of Examples\", int(count[0]))\n",
"\n",
"print(\"First 10 rows\")\n",
"! gcloud storage cat $FILE | head" ]
"! gsutil cat $FILE | head"
]
},
{
"cell_type": "markdown",
@@ -1321,7 +1325,8 @@
"source": [
"import json\n",
"\n",
"test_items = !gcloud storage cat $IMPORT_FILE | head -n2\n", "test_data_1 = test_items[0].replace(\"'\", '\"')\n",
"test_items = !gsutil cat $IMPORT_FILE | head -n2\n",
"test_data_1 = test_items[0].replace(\"'\", '\"')\n",
"test_data_1 = json.loads(test_data_1)\n",
"test_data_2 = test_items[0].replace(\"'\", '\"')\n",
"test_data_2 = json.loads(test_data_2)\n",
@@ -1362,8 +1367,9 @@
"file_1 = test_item_1.split(\"/\")[-1]\n",
"file_2 = test_item_2.split(\"/\")[-1]\n",
"\n",
"! gcloud storage cp $test_item_1 $BUCKET_NAME/$file_1\n",
"! gcloud storage cp $test_item_2 $BUCKET_NAME/$file_2\n", "\n",
"! gsutil cp $test_item_1 $BUCKET_NAME/$file_1\n",
"! gsutil cp $test_item_2 $BUCKET_NAME/$file_2\n",
"\n",
"test_item_1 = BUCKET_NAME + \"/\" + file_1\n",
"test_item_2 = BUCKET_NAME + \"/\" + file_2"
]
@@ -1406,7 +1412,8 @@
" f.write(json.dumps(data) + \"\\n\")\n",
"\n",
"print(gcs_input_uri)\n",
"! gcloud storage cat $gcs_input_uri" ]
"! gsutil cat $gcs_input_uri"
]
},
{
"cell_type": "markdown",
@@ -1674,7 +1681,8 @@
"source": [
"def get_latest_predictions(gcs_out_dir):\n",
" \"\"\" Get the latest prediction subfolder using the timestamp in the subfolder name\"\"\"\n",
" folders = !gcloud storage ls $gcs_out_dir\n", " latest = \"\"\n",
" folders = !gsutil ls $gcs_out_dir\n",
" latest = \"\"\n",
" for folder in folders:\n",
" subfolder = folder.split(\"/\")[-2]\n",
" if subfolder.startswith(\"prediction-\"):\n",
@@ -1691,8 +1699,10 @@
" raise Exception(\"Batch Job Failed\")\n",
" else:\n",
" folder = get_latest_predictions(predictions)\n",
" ! gcloud storage ls $folder/prediction*.jsonl\n", "\n",
" ! gcloud storage cat $folder/prediction*.jsonl\n", " break\n",
" ! gsutil ls $folder/prediction*.jsonl\n",
"\n",
" ! gsutil cat $folder/prediction*.jsonl\n",
" break\n",
" time.sleep(60)"
]
},
@@ -1786,7 +1796,8 @@
" print(e)\n",
"\n",
"if delete_bucket and \"BUCKET_NAME\" in globals():\n",
" ! gcloud storage rm --recursive $BUCKET_NAME" ]
" ! gsutil rm -r $BUCKET_NAME"
]
}
],
"metadata": {
@@ -39,7 +39,7 @@
" </td>\n",
" <td>\n",
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/gapic/automl/showcase_automl_image_segmentation_online.ipynb\">\n",
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
" View on GitHub\n",
" </a>\n",
" </td>\n",
@@ -726,10 +726,12 @@
"else:\n",
" FILE = IMPORT_FILE\n",
"\n",
"count = ! gcloud storage cat $FILE | wc -l\n", "print(\"Number of Examples\", int(count[0]))\n",
"count = ! gsutil cat $FILE | wc -l\n",
"print(\"Number of Examples\", int(count[0]))\n",
"\n",
"print(\"First 10 rows\")\n",
"! gcloud storage cat $FILE | head" ]
"! gsutil cat $FILE | head"
]
},
{
"cell_type": "markdown",
@@ -1336,7 +1338,8 @@
"source": [
"import json\n",
"\n",
"test_items = !gcloud storage cat $IMPORT_FILE | head -n1\n", "test_data = test_items[0].replace(\"'\", '\"')\n",
"test_items = !gsutil cat $IMPORT_FILE | head -n1\n",
"test_data = test_items[0].replace(\"'\", '\"')\n",
"test_data = json.loads(test_data)\n",
"try:\n",
" test_item = test_data[\"image_gcs_uri\"]\n",
@@ -1552,7 +1555,8 @@
" print(e)\n",
"\n",
"if delete_bucket and \"BUCKET_NAME\" in globals():\n",
" ! gcloud storage rm --recursive $BUCKET_NAME" ]
" ! gsutil rm -r $BUCKET_NAME"
]
}
],
"metadata": {
@@ -39,7 +39,7 @@
" </td>\n",
" <td>\n",
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/gapic/automl/showcase_automl_tabular_binary_classification_batch.ipynb\">\n",
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
" View on GitHub\n",
" </a>\n",
" </td>\n",
@@ -421,7 +421,8 @@
},
"outputs": [],
"source": [
"! gcloud storage buckets create --location $REGION $BUCKET_NAME" ]
"! gsutil mb -l $REGION $BUCKET_NAME"
]
},
{
"cell_type": "markdown",
@@ -440,7 +441,8 @@
},
"outputs": [],
"source": [
"! gcloud storage ls --all-versions --long $BUCKET_NAME" ]
"! gsutil ls -al $BUCKET_NAME"
]
},
{
"cell_type": "markdown",
@@ -807,11 +809,14 @@
},
"outputs": [],
"source": [
"count = ! gcloud storage cat $IMPORT_FILE | wc -l\n", "print(\"Number of Examples\", int(count[0]))\n",
"count = ! gsutil cat $IMPORT_FILE | wc -l\n",
"print(\"Number of Examples\", int(count[0]))\n",
"\n",
"print(\"First 10 rows\")\n",
"! gcloud storage cat $IMPORT_FILE | head\n", "\n",
"heading = ! gcloud storage cat $IMPORT_FILE | head -n1\n", "label_column = str(heading).split(\",\")[-1].split(\"'\")[0]\n",
"! gsutil cat $IMPORT_FILE | head\n",
"\n",
"heading = ! gsutil cat $IMPORT_FILE | head -n1\n",
"label_column = str(heading).split(\",\")[-1].split(\"'\")[0]\n",
"print(\"Label Column Name\", label_column)\n",
"if label_column is None:\n",
" raise Exception(\"label column missing\")"
@@ -1372,7 +1377,8 @@
" f.write(str(INSTANCE_2) + \"\\n\")\n",
"\n",
"print(gcs_input_uri)\n",
"! gcloud storage cat $gcs_input_uri" ]
"! gsutil cat $gcs_input_uri"
]
},
{
"cell_type": "markdown",
@@ -1640,7 +1646,8 @@
"source": [
"def get_latest_predictions(gcs_out_dir):\n",
" \"\"\" Get the latest prediction subfolder using the timestamp in the subfolder name\"\"\"\n",
" folders = !gcloud storage ls $gcs_out_dir\n", " latest = \"\"\n",
" folders = !gsutil ls $gcs_out_dir\n",
" latest = \"\"\n",
" for folder in folders:\n",
" subfolder = folder.split(\"/\")[-2]\n",
" if subfolder.startswith(\"prediction-\"):\n",
@@ -1657,8 +1664,10 @@
" raise Exception(\"Batch Job Failed\")\n",
" else:\n",
" folder = get_latest_predictions(predictions)\n",
" ! gcloud storage ls $folder/prediction*.csv\n", "\n",
" ! gcloud storage cat $folder/prediction*.csv\n", " break\n",
" ! gsutil ls $folder/prediction*.csv\n",
"\n",
" ! gsutil cat $folder/prediction*.csv\n",
" break\n",
" time.sleep(60)"
]
},
@@ -1752,7 +1761,8 @@
" print(e)\n",
"\n",
"if delete_bucket and \"BUCKET_NAME\" in globals():\n",
" ! gcloud storage rm --recursive $BUCKET_NAME" ]
" ! gsutil rm -r $BUCKET_NAME"
]
}
],
"metadata": {
@@ -39,7 +39,7 @@
" </td>\n",
" <td>\n",
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/gapic/automl/showcase_automl_tabular_binary_classification_online.ipynb\">\n",
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
" View on GitHub\n",
" </a>\n",
" </td>\n",
@@ -735,11 +735,14 @@
},
"outputs": [],
"source": [
"count = ! gcloud storage cat $IMPORT_FILE | wc -l\n", "print(\"Number of Examples\", int(count[0]))\n",
"count = ! gsutil cat $IMPORT_FILE | wc -l\n",
"print(\"Number of Examples\", int(count[0]))\n",
"\n",
"print(\"First 10 rows\")\n",
"! gcloud storage cat $IMPORT_FILE | head\n", "\n",
"heading = ! gcloud storage cat $IMPORT_FILE | head -n1\n", "label_column = str(heading).split(\",\")[-1].split(\"'\")[0]\n",
"! gsutil cat $IMPORT_FILE | head\n",
"\n",
"heading = ! gsutil cat $IMPORT_FILE | head -n1\n",
"label_column = str(heading).split(\",\")[-1].split(\"'\")[0]\n",
"print(\"Label Column Name\", label_column)\n",
"if label_column is None:\n",
" raise Exception(\"label column missing\")"
@@ -1665,7 +1668,8 @@
" print(e)\n",
"\n",
"if delete_bucket and \"BUCKET_NAME\" in globals():\n",
" ! gcloud storage rm --recursive $BUCKET_NAME" ]
" ! gsutil rm -r $BUCKET_NAME"
]
}
],
"metadata": {
@@ -39,7 +39,7 @@
" </td>\n",
" <td>\n",
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/gapic/automl/showcase_automl_tabular_classification_batch.ipynb\">\n",
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
" View on GitHub\n",
" </a>\n",
" </td>\n",
@@ -421,7 +421,8 @@
},
"outputs": [],
"source": [
"! gcloud storage buckets create --location $REGION $BUCKET_NAME" ]
"! gsutil mb -l $REGION $BUCKET_NAME"
]
},
{
"cell_type": "markdown",
@@ -440,7 +441,8 @@
},
"outputs": [],
"source": [
"! gcloud storage ls --all-versions --long $BUCKET_NAME" ]
"! gsutil ls -al $BUCKET_NAME"
]
},
{
"cell_type": "markdown",
@@ -808,11 +810,14 @@
},
"outputs": [],
"source": [
"count = ! gcloud storage cat $IMPORT_FILE | wc -l\n", "print(\"Number of Examples\", int(count[0]))\n",
"count = ! gsutil cat $IMPORT_FILE | wc -l\n",
"print(\"Number of Examples\", int(count[0]))\n",
"\n",
"print(\"First 10 rows\")\n",
"! gcloud storage cat $IMPORT_FILE | head\n", "\n",
"heading = ! gcloud storage cat $IMPORT_FILE | head -n1\n", "label_column = str(heading).split(\",\")[-1].split(\"'\")[0]\n",
"! gsutil cat $IMPORT_FILE | head\n",
"\n",
"heading = ! gsutil cat $IMPORT_FILE | head -n1\n",
"label_column = str(heading).split(\",\")[-1].split(\"'\")[0]\n",
"print(\"Label Column Name\", label_column)\n",
"if label_column is None:\n",
" raise Exception(\"label column missing\")"
@@ -1358,7 +1363,8 @@
" f.write(str(INSTANCE_2) + \"\\n\")\n",
"\n",
"print(gcs_input_uri)\n",
"! gcloud storage cat $gcs_input_uri" ]
"! gsutil cat $gcs_input_uri"
]
},
{
"cell_type": "markdown",
@@ -1626,7 +1632,8 @@
"source": [
"def get_latest_predictions(gcs_out_dir):\n",
" \"\"\" Get the latest prediction subfolder using the timestamp in the subfolder name\"\"\"\n",
" folders = !gcloud storage ls $gcs_out_dir\n", " latest = \"\"\n",
" folders = !gsutil ls $gcs_out_dir\n",
" latest = \"\"\n",
" for folder in folders:\n",
" subfolder = folder.split(\"/\")[-2]\n",
" if subfolder.startswith(\"prediction-\"):\n",
@@ -1643,8 +1650,10 @@
" raise Exception(\"Batch Job Failed\")\n",
" else:\n",
" folder = get_latest_predictions(predictions)\n",
" ! gcloud storage ls $folder/prediction*.csv\n", "\n",
" ! gcloud storage cat $folder/prediction*.csv\n", " break\n",
" ! gsutil ls $folder/prediction*.csv\n",
"\n",
" ! gsutil cat $folder/prediction*.csv\n",
" break\n",
" time.sleep(60)"
]
},
@@ -1738,7 +1747,8 @@
" print(e)\n",
"\n",
"if delete_bucket and \"BUCKET_NAME\" in globals():\n",
" ! gcloud storage rm --recursive $BUCKET_NAME" ]
" ! gsutil rm -r $BUCKET_NAME"
]
}
],
"metadata": {
@@ -39,7 +39,7 @@
" </td>\n",
" <td>\n",
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/gapic/automl/showcase_automl_tabular_classification_batch_explain.ipynb\">\n",
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
" View on GitHub\n",
" </a>\n",
" </td>\n",
@@ -421,7 +421,8 @@
},
"outputs": [],
"source": [
"! gcloud storage buckets create --location $REGION $BUCKET_NAME" ]
"! gsutil mb -l $REGION $BUCKET_NAME"
]
},
{
"cell_type": "markdown",
@@ -440,7 +441,8 @@
},
"outputs": [],
"source": [
"! gcloud storage ls --all-versions --long $BUCKET_NAME" ]
"! gsutil ls -al $BUCKET_NAME"
]
},
{
"cell_type": "markdown",
@@ -808,11 +810,14 @@
},
"outputs": [],
"source": [
"count = ! gcloud storage cat $IMPORT_FILE | wc -l\n", "print(\"Number of Examples\", int(count[0]))\n",
"count = ! gsutil cat $IMPORT_FILE | wc -l\n",
"print(\"Number of Examples\", int(count[0]))\n",
"\n",
"print(\"First 10 rows\")\n",
"! gcloud storage cat $IMPORT_FILE | head\n", "\n",
"heading = ! gcloud storage cat $IMPORT_FILE | head -n1\n", "label_column = str(heading).split(\",\")[-1].split(\"'\")[0]\n",
"! gsutil cat $IMPORT_FILE | head\n",
"\n",
"heading = ! gsutil cat $IMPORT_FILE | head -n1\n",
"label_column = str(heading).split(\",\")[-1].split(\"'\")[0]\n",
"print(\"Label Column Name\", label_column)\n",
"if label_column is None:\n",
" raise Exception(\"label column missing\")"
@@ -1358,7 +1363,8 @@
" f.write(str(INSTANCE_2) + \"\\n\")\n",
"\n",
"print(gcs_input_uri)\n",
"! gcloud storage cat $gcs_input_uri" ]
"! gsutil cat $gcs_input_uri"
]
},
{
"cell_type": "markdown",
@@ -1625,7 +1631,8 @@
"source": [
"def get_latest_predictions(gcs_out_dir):\n",
" \"\"\" Get the latest prediction subfolder using the timestamp in the subfolder name\"\"\"\n",
" folders = !gcloud storage ls $gcs_out_dir\n", " latest = \"\"\n",
" folders = !gsutil ls $gcs_out_dir\n",
" latest = \"\"\n",
" for folder in folders:\n",
" subfolder = folder.split(\"/\")[-2]\n",
" if subfolder.startswith(\"prediction-\"):\n",
@@ -1642,8 +1649,10 @@
" raise Exception(\"Batch Job Failed\")\n",
" else:\n",
" folder = get_latest_predictions(predictions)\n",
" ! gcloud storage ls $folder/explanation*.csv\n", "\n",
" ! gcloud storage cat $folder/explanation*.csv\n", " break\n",
" ! gsutil ls $folder/explanation*.csv\n",
"\n",
" ! gsutil cat $folder/explanation*.csv\n",
" break\n",
" time.sleep(60)"
]
},
@@ -1737,7 +1746,8 @@
" print(e)\n",
"\n",
"if delete_bucket and \"BUCKET_NAME\" in globals():\n",
" ! gcloud storage rm --recursive $BUCKET_NAME" ]
" ! gsutil rm -r $BUCKET_NAME"
]
}
],
"metadata": {
@@ -39,7 +39,7 @@
" </td>\n",
" <td>\n",
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/gapic/automl/showcase_automl_tabular_classification_export_cloud.ipynb\">\n",
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
" View on GitHub\n",
" </a>\n",
" </td>\n",
@@ -416,7 +416,8 @@
},
"outputs": [],
"source": [
"! gcloud storage buckets create --location $REGION $BUCKET_NAME" ]
"! gsutil mb -l $REGION $BUCKET_NAME"
]
},
{
"cell_type": "markdown",
@@ -435,7 +436,8 @@
},
"outputs": [],
"source": [
"! gcloud storage ls --all-versions --long $BUCKET_NAME" ]
"! gsutil ls -al $BUCKET_NAME"
]
},
{
"cell_type": "markdown",
@@ -747,11 +749,14 @@
},
"outputs": [],
"source": [
"count = ! gcloud storage cat $IMPORT_FILE | wc -l\n", "print(\"Number of Examples\", int(count[0]))\n",
"count = ! gsutil cat $IMPORT_FILE | wc -l\n",
"print(\"Number of Examples\", int(count[0]))\n",
"\n",
"print(\"First 10 rows\")\n",
"! gcloud storage cat $IMPORT_FILE | head\n", "\n",
"heading = ! gcloud storage cat $IMPORT_FILE | head -n1\n", "label_column = str(heading).split(\",\")[-1].split(\"'\")[0]\n",
"! gsutil cat $IMPORT_FILE | head\n",
"\n",
"heading = ! gsutil cat $IMPORT_FILE | head -n1\n",
"label_column = str(heading).split(\",\")[-1].split(\"'\")[0]\n",
"print(\"Label Column Name\", label_column)\n",
"if label_column is None:\n",
" raise Exception(\"label column missing\")"
@@ -1278,12 +1283,15 @@
"source": [
"print(\"Model Package:\", model_package)\n",
"print(\"Contents:\")\n",
"! gcloud storage ls $model_package\n", "\n",
"! gsutil ls $model_package\n",
"\n",
"print(\"\\nTF Saved Model\")\n",
"path = model_package + \"/predict\"\n",
"files = ! gcloud storage ls $path\n", "saved_dir = files[1]\n",
"files = ! gsutil ls $path\n",
"saved_dir = files[1]\n",
"print(saved_dir)\n",
"! gcloud storage ls $saved_dir" ]
"! gsutil ls $saved_dir"
]
},
{
"cell_type": "markdown",
@@ -1304,7 +1312,8 @@
},
"outputs": [],
"source": [
"! gcloud storage cp --recursive $model_package ." ]
"! gsutil cp -r $model_package ."
]
},
{
"cell_type": "markdown",
@@ -1590,7 +1599,8 @@
" print(e)\n",
"\n",
"if delete_bucket and \"BUCKET_NAME\" in globals():\n",
" ! gcloud storage rm --recursive $BUCKET_NAME" ]
" ! gsutil rm -r $BUCKET_NAME"
]
}
],
"metadata": {
@@ -39,7 +39,7 @@
" </td>\n",
" <td>\n",
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/gapic/automl/showcase_automl_tabular_classification_online.ipynb\">\n",
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
" View on GitHub\n",
" </a>\n",
" </td>\n",
@@ -735,11 +735,14 @@
},
"outputs": [],
"source": [
"count = ! gcloud storage cat $IMPORT_FILE | wc -l\n", "print(\"Number of Examples\", int(count[0]))\n",
"count = ! gsutil cat $IMPORT_FILE | wc -l\n",
"print(\"Number of Examples\", int(count[0]))\n",
"\n",
"print(\"First 10 rows\")\n",
"! gcloud storage cat $IMPORT_FILE | head\n", "\n",
"heading = ! gcloud storage cat $IMPORT_FILE | head -n1\n", "label_column = str(heading).split(\",\")[-1].split(\"'\")[0]\n",
"! gsutil cat $IMPORT_FILE | head\n",
"\n",
"heading = ! gsutil cat $IMPORT_FILE | head -n1\n",
"label_column = str(heading).split(\",\")[-1].split(\"'\")[0]\n",
"print(\"Label Column Name\", label_column)\n",
"if label_column is None:\n",
" raise Exception(\"label column missing\")"
@@ -1642,7 +1645,8 @@
" print(e)\n",
"\n",
"if delete_bucket and \"BUCKET_NAME\" in globals():\n",
" ! gcloud storage rm --recursive $BUCKET_NAME" ]
" ! gsutil rm -r $BUCKET_NAME"
]
}
],
"metadata": {
@@ -39,7 +39,7 @@
" </td>\n",
" <td>\n",
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/gapic/automl/showcase_automl_tabular_classification_online_explain.ipynb\">\n",
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
" View on GitHub\n",
" </a>\n",
" </td>\n",
@@ -397,6 +397,7 @@
"\n",
"import google.cloud.aiplatform_v1beta1 as aip\n",
"from google.protobuf import json_format\n",
"from google.protobuf.json_format import MessageToJson, ParseDict\n",
"from google.protobuf.struct_pb2 import Struct, Value"
]
},
@@ -734,13 +735,13 @@
},
"outputs": [],
"source": [
"count = ! gcloud storage cat $IMPORT_FILE | wc -l\n",
"count = ! gsutil cat $IMPORT_FILE | wc -l\n",
"print(\"Number of Examples\", int(count[0]))\n",
"\n",
"print(\"First 10 rows\")\n",
"! gcloud storage cat $IMPORT_FILE | head\n",
"! gsutil cat $IMPORT_FILE | head\n",
"\n",
"heading = ! gcloud storage cat $IMPORT_FILE | head -n1\n",
"heading = ! gsutil cat $IMPORT_FILE | head -n1\n",
"label_column = str(heading).split(\",\")[-1].split(\"'\")[0]\n",
"print(\"Label Column Name\", label_column)\n",
"if label_column is None:\n",
@@ -1819,7 +1820,7 @@
" print(e)\n",
"\n",
"if delete_bucket and \"BUCKET_NAME\" in globals():\n",
" ! gcloud storage rm --recursive $BUCKET_NAME"
" ! gsutil rm -r $BUCKET_NAME"
]
}
],
@@ -39,7 +39,7 @@
" </td>\n",
" <td>\n",
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/gapic/automl/showcase_automl_tabular_forecasting_batch.ipynb\">\n",
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
" View on GitHub\n",
" </a>\n",
" </td>\n",
@@ -421,7 +421,8 @@
},
"outputs": [],
"source": [
"! gcloud storage buckets create --location=$REGION $BUCKET_NAME" ]
"! gsutil mb -l $REGION $BUCKET_NAME"
]
},
{
"cell_type": "markdown",
@@ -440,7 +441,8 @@
},
"outputs": [],
"source": [
"! gcloud storage ls --all-versions --long $BUCKET_NAME" ]
"! gsutil ls -al $BUCKET_NAME"
]
},
{
"cell_type": "markdown",
@@ -777,11 +779,14 @@
},
"outputs": [],
"source": [
"count = ! gcloud storage cat $IMPORT_FILE | wc -l\n", "print(\"Number of Examples\", int(count[0]))\n",
"count = ! gsutil cat $IMPORT_FILE | wc -l\n",
"print(\"Number of Examples\", int(count[0]))\n",
"\n",
"print(\"First 10 rows\")\n",
"! gcloud storage cat $IMPORT_FILE | head\n", "\n",
"heading = ! gcloud storage cat $IMPORT_FILE | head -n1\n", "label_column = \"deaths\" # @param {type:\"string\"}\n",
"! gsutil cat $IMPORT_FILE | head\n",
"\n",
"heading = ! gsutil cat $IMPORT_FILE | head -n1\n",
"label_column = \"deaths\" # @param {type:\"string\"}\n",
"time_column = \"date\" # @param {type:\"string\"}\n",
"time_series_identifier_column = \"county\" # @param {type:\"string\"}\n",
"print(\"Label Column Name\", label_column)\n",
@@ -1353,7 +1358,8 @@
" f.write(str(INSTANCE_2) + \"\\n\")\n",
"\n",
"print(gcs_input_uri)\n",
"! gcloud storage cat $gcs_input_uri" ]
"! gsutil cat $gcs_input_uri"
]
},
{
"cell_type": "markdown",
@@ -1621,7 +1627,8 @@
"source": [
"def get_latest_predictions(gcs_out_dir):\n",
" \"\"\" Get the latest prediction subfolder using the timestamp in the subfolder name\"\"\"\n",
" folders = !gcloud storage ls $gcs_out_dir\n", " latest = \"\"\n",
" folders = !gsutil ls $gcs_out_dir\n",
" latest = \"\"\n",
" for folder in folders:\n",
" subfolder = folder.split(\"/\")[-2]\n",
" if subfolder.startswith(\"prediction-\"):\n",
@@ -1638,8 +1645,10 @@
" raise Exception(\"Batch Job Failed\")\n",
" else:\n",
" folder = get_latest_predictions(predictions)\n",
" ! gcloud storage ls $folder/prediction*.csv\n", "\n",
" ! gcloud storage cat $folder/prediction*.csv\n", " break\n",
" ! gsutil ls $folder/prediction*.csv\n",
"\n",
" ! gsutil cat $folder/prediction*.csv\n",
" break\n",
" time.sleep(60)"
]
},
@@ -1733,7 +1742,8 @@
" print(e)\n",
"\n",
"if delete_bucket and \"BUCKET_NAME\" in globals():\n",
" ! gcloud storage rm --recursive $BUCKET_NAME" ]
" ! gsutil rm -r $BUCKET_NAME"
]
}
],
"metadata": {
@@ -39,7 +39,7 @@
" </td>\n",
" <td>\n",
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/gapic/automl/showcase_automl_tabular_regression_online_bq.ipynb\">\n",
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
" View on GitHub\n",
" </a>\n",
" </td>\n",
@@ -397,6 +397,7 @@
"\n",
"from google.cloud.aiplatform import gapic as aip\n",
"from google.protobuf import json_format\n",
"from google.protobuf.json_format import MessageToJson, ParseDict\n",
"from google.protobuf.struct_pb2 import Struct, Value"
]
},
@@ -1613,7 +1614,7 @@
" print(e)\n",
"\n",
"if delete_bucket and \"BUCKET_NAME\" in globals():\n",
" ! gcloud storage rm --recursive $BUCKET_NAME"
" ! gsutil rm -r $BUCKET_NAME"
]
}
],
@@ -39,7 +39,7 @@
" </td>\n",
" <td>\n",
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/gapic/automl/showcase_automl_text_classification_batch.ipynb\">\n",
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
" View on GitHub\n",
" </a>\n",
" </td>\n",
@@ -421,7 +421,8 @@
},
"outputs": [],
"source": [
"! gcloud storage buckets create --location $REGION $BUCKET_NAME" ]
"! gsutil mb -l $REGION $BUCKET_NAME"
]
},
{
"cell_type": "markdown",
@@ -440,7 +441,8 @@
},
"outputs": [],
"source": [
"! gcloud storage ls --all-versions --long $BUCKET_NAME" ]
"! gsutil ls -al $BUCKET_NAME"
]
},
{
"cell_type": "markdown",
@@ -879,10 +881,12 @@
"else:\n",
" FILE = IMPORT_FILE\n",
"\n",
"count = ! gcloud storage cat $FILE | wc -l\n", "print(\"Number of Examples\", int(count[0]))\n",
"count = ! gsutil cat $FILE | wc -l\n",
"print(\"Number of Examples\", int(count[0]))\n",
"\n",
"print(\"First 10 rows\")\n",
"! gcloud storage cat $FILE | head" ]
"! gsutil cat $FILE | head"
]
},
{
"cell_type": "markdown",
@@ -1310,7 +1314,8 @@
},
"outputs": [],
"source": [
"test_items = ! gcloud storage cat $IMPORT_FILE | head -n2\n", "if len(test_items[0]) == 3:\n",
"test_items = ! gsutil cat $IMPORT_FILE | head -n2\n",
"if len(test_items[0]) == 3:\n",
" _, test_item_1, test_label_1 = str(test_items[0]).split(\",\")\n",
" _, test_item_2, test_label_2 = str(test_items[1]).split(\",\")\n",
"else:\n",
@@ -1366,7 +1371,8 @@
" f.write(json.dumps(data) + \"\\n\")\n",
"\n",
"print(gcs_input_uri)\n",
"! gcloud storage cat $gcs_input_uri" ]
"! gsutil cat $gcs_input_uri"
]
},
{
"cell_type": "markdown",
@@ -1638,7 +1644,8 @@
"source": [
"def get_latest_predictions(gcs_out_dir):\n",
" \"\"\" Get the latest prediction subfolder using the timestamp in the subfolder name\"\"\"\n",
" folders = !gcloud storage ls $gcs_out_dir\n", " latest = \"\"\n",
" folders = !gsutil ls $gcs_out_dir\n",
" latest = \"\"\n",
" for folder in folders:\n",
" subfolder = folder.split(\"/\")[-2]\n",
" if subfolder.startswith(\"prediction-\"):\n",
@@ -1655,8 +1662,10 @@
" raise Exception(\"Batch Job Failed\")\n",
" else:\n",
" folder = get_latest_predictions(predictions)\n",
" ! gcloud storage ls $folder/prediction*.jsonl\n", "\n",
! gcloud storage cat $folder/prediction*.jsonl\n", " break\n",
" ! gsutil ls $folder/prediction*.jsonl\n",
"\n",
" ! gsutil cat $folder/prediction*.jsonl\n",
" break\n",
" time.sleep(60)"
]
},
@@ -1750,7 +1759,8 @@
" print(e)\n",
"\n",
"if delete_bucket and \"BUCKET_NAME\" in globals():\n",
" ! gcloud storage rm --recursive $BUCKET_NAME" ]
" ! gsutil rm -r $BUCKET_NAME"
]
}
],
"metadata": {
@@ -39,7 +39,7 @@
" </td>\n",
" <td>\n",
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/gapic/automl/showcase_automl_text_classification_online.ipynb\">\n",
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
" View on GitHub\n",
" </a>\n",
" </td>\n",
@@ -716,10 +716,12 @@
"else:\n",
" FILE = IMPORT_FILE\n",
"\n",
"count = ! gcloud storage cat $FILE | wc -l\n", "print(\"Number of Examples\", int(count[0]))\n",
"count = ! gsutil cat $FILE | wc -l\n",
"print(\"Number of Examples\", int(count[0]))\n",
"\n",
"print(\"First 10 rows\")\n",
"! gcloud storage cat $FILE | head" ]
"! gsutil cat $FILE | head"
]
},
{
"cell_type": "markdown",
@@ -1325,7 +1327,7 @@
},
"outputs": [],
"source": [
"test_item = ! gcloud storage cat $IMPORT_FILE | head -n1\n",
"test_item = ! gsutil cat $IMPORT_FILE | head -n1\n",
"if len(test_item[0]) == 3:\n",
" _, test_item, test_label = str(test_item[0]).split(\",\")\n",
"else:\n",
@@ -1531,7 +1533,8 @@
" print(e)\n",
"\n",
"if delete_bucket and \"BUCKET_NAME\" in globals():\n",
" ! gcloud storage rm --recursive $BUCKET_NAME" ]
" ! gsutil rm -r $BUCKET_NAME"
]
}
],
"metadata": {
@@ -39,7 +39,7 @@
" </td>\n",
" <td>\n",
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/gapic/automl/showcase_automl_text_entity_extraction_batch.ipynb\">\n",
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
" View on GitHub\n",
" </a>\n",
" </td>\n",
@@ -421,7 +421,7 @@
},
"outputs": [],
"source": [
"! gcloud storage buckets create --location $REGION $BUCKET_NAME"
"! gsutil mb -l $REGION $BUCKET_NAME"
]
},
{
@@ -441,7 +441,7 @@
},
"outputs": [],
"source": [
"! gcloud storage ls --all-versions --long $BUCKET_NAME"
"! gsutil ls -al $BUCKET_NAME"
]
},
{
@@ -478,7 +478,9 @@
"import time\n",
"\n",
"from google.cloud.aiplatform import gapic as aip\n",
"from google.protobuf import json_format"
"from google.protobuf import json_format\n",
"from google.protobuf.json_format import MessageToJson, ParseDict\n",
"from google.protobuf.struct_pb2 import Struct, Value"
]
},
{
@@ -886,11 +888,11 @@
"else:\n",
" FILE = IMPORT_FILE\n",
"\n",
"count = ! gcloud storage cat $FILE | wc -l\n",
"count = ! gsutil cat $FILE | wc -l\n",
"print(\"Number of Examples\", int(count[0]))\n",
"\n",
"print(\"First 10 rows\")\n",
"! gcloud storage cat $FILE | head"
"! gsutil cat $FILE | head"
]
},
{
@@ -1375,7 +1377,7 @@
" f.write(json.dumps(data) + \"\\n\")\n",
"\n",
"print(gcs_input_uri)\n",
"! gcloud storage cat $gcs_input_uri"
"! gsutil cat $gcs_input_uri"
]
},
{
@@ -1650,8 +1652,8 @@
"outputs": [],
"source": [
"def get_latest_predictions(gcs_out_dir):\n",
" \"\"\"Get the latest prediction subfolder using the timestamp in the subfolder name\"\"\"\n",
" folders = !gcloud storage ls $gcs_out_dir\n",
" \"\"\" Get the latest prediction subfolder using the timestamp in the subfolder name\"\"\"\n",
" folders = !gsutil ls $gcs_out_dir\n",
" latest = \"\"\n",
" for folder in folders:\n",
" subfolder = folder.split(\"/\")[-2]\n",
@@ -1669,9 +1671,9 @@
" raise Exception(\"Batch Job Failed\")\n",
" else:\n",
" folder = get_latest_predictions(predictions)\n",
" ! gcloud storage ls $folder/prediction*.jsonl\n",
" ! gsutil ls $folder/prediction*.jsonl\n",
"\n",
" ! gcloud storage cat $folder/prediction*.jsonl\n",
" ! gsutil cat $folder/prediction*.jsonl\n",
" break\n",
" time.sleep(60)"
]
@@ -1766,7 +1768,7 @@
" print(e)\n",
"\n",
"if delete_bucket and \"BUCKET_NAME\" in globals():\n",
" ! gcloud storage rm --recursive $BUCKET_NAME"
" ! gsutil rm -r $BUCKET_NAME"
]
}
],
@@ -39,7 +39,7 @@
" </td>\n",
" <td>\n",
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/gapic/automl/showcase_automl_text_entity_extraction_online.ipynb\">\n",
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
" View on GitHub\n",
" </a>\n",
" </td>\n",
@@ -723,10 +723,12 @@
"else:\n",
" FILE = IMPORT_FILE\n",
"\n",
"count = ! gcloud storage cat $FILE | wc -l\n", "print(\"Number of Examples\", int(count[0]))\n",
"count = ! gsutil cat $FILE | wc -l\n",
"print(\"Number of Examples\", int(count[0]))\n",
"\n",
"print(\"First 10 rows\")\n",
"! gcloud storage cat $FILE | head" ]
"! gsutil cat $FILE | head"
]
},
{
"cell_type": "markdown",
@@ -1542,7 +1544,8 @@
" print(e)\n",
"\n",
"if delete_bucket and \"BUCKET_NAME\" in globals():\n",
" ! gcloud storage rm --recursive $BUCKET_NAME" ]
" ! gsutil rm -r $BUCKET_NAME"
]
}
],
"metadata": {
@@ -39,7 +39,7 @@
" </td>\n",
" <td>\n",
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/gapic/automl/showcase_automl_text_multi-label_classification_batch.ipynb\">\n",
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
" View on GitHub\n",
" </a>\n",
" </td>\n",
@@ -421,7 +421,8 @@
},
"outputs": [],
"source": [
"! gcloud storage buckets create --location=$REGION $BUCKET_NAME" ]
"! gsutil mb -l $REGION $BUCKET_NAME"
]
},
{
"cell_type": "markdown",
@@ -440,7 +441,8 @@
},
"outputs": [],
"source": [
"! gcloud storage ls --all-versions --long $BUCKET_NAME" ]
"! gsutil ls -al $BUCKET_NAME"
]
},
{
"cell_type": "markdown",
@@ -879,10 +881,12 @@
"else:\n",
" FILE = IMPORT_FILE\n",
"\n",
"count = ! gcloud storage cat $FILE | wc -l\n", "print(\"Number of Examples\", int(count[0]))\n",
"count = ! gsutil cat $FILE | wc -l\n",
"print(\"Number of Examples\", int(count[0]))\n",
"\n",
"print(\"First 10 rows\")\n",
"! gcloud storage cat $FILE | head" ]
"! gsutil cat $FILE | head"
]
},
{
"cell_type": "markdown",
@@ -1310,7 +1314,8 @@
},
"outputs": [],
"source": [
"test_items = ! gcloud storage cat $IMPORT_FILE | head -n2\n", "\n",
"test_items = ! gsutil cat $IMPORT_FILE | head -n2\n",
"\n",
"cols_1 = str(test_items[0]).split(\",\")\n",
"cols_2 = str(test_items[1]).split(\",\")\n",
"test_item_1 = cols_1[0]\n",
@@ -1367,7 +1372,8 @@
" f.write(json.dumps(data) + \"\\n\")\n",
"\n",
"print(gcs_input_uri)\n",
"! gcloud storage cat $gcs_input_uri" ]
"! gsutil cat $gcs_input_uri"
]
},
{
"cell_type": "markdown",
@@ -1639,7 +1645,8 @@
"source": [
"def get_latest_predictions(gcs_out_dir):\n",
" \"\"\" Get the latest prediction subfolder using the timestamp in the subfolder name\"\"\"\n",
" folders = !gcloud storage ls $gcs_out_dir\n", " latest = \"\"\n",
" folders = !gsutil ls $gcs_out_dir\n",
" latest = \"\"\n",
" for folder in folders:\n",
" subfolder = folder.split(\"/\")[-2]\n",
" if subfolder.startswith(\"prediction-\"):\n",
@@ -1656,8 +1663,10 @@
" raise Exception(\"Batch Job Failed\")\n",
" else:\n",
" folder = get_latest_predictions(predictions)\n",
" ! gcloud storage ls $folder/prediction*.jsonl\n", "\n",
" ! gcloud storage cat $folder/prediction*.jsonl\n", " break\n",
" ! gsutil ls $folder/prediction*.jsonl\n",
"\n",
" ! gsutil cat $folder/prediction*.jsonl\n",
" break\n",
" time.sleep(60)"
]
},
@@ -1751,7 +1760,8 @@
" print(e)\n",
"\n",
"if delete_bucket and \"BUCKET_NAME\" in globals():\n",
" ! gcloud storage rm --recursive $BUCKET_NAME" ]
" ! gsutil rm -r $BUCKET_NAME"
]
}
],
"metadata": {
@@ -39,7 +39,7 @@
" </td>\n",
" <td>\n",
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/gapic/automl/showcase_automl_text_multi-label_classification_online.ipynb\">\n",
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
" View on GitHub\n",
" </a>\n",
" </td>\n",
@@ -716,10 +716,12 @@
"else:\n",
" FILE = IMPORT_FILE\n",
"\n",
"count = ! gcloud storage cat $FILE | wc -l\n", "print(\"Number of Examples\", int(count[0]))\n",
"count = ! gsutil cat $FILE | wc -l\n",
"print(\"Number of Examples\", int(count[0]))\n",
"\n",
"print(\"First 10 rows\")\n",
"! gcloud storage cat $FILE | head" ]
"! gsutil cat $FILE | head"
]
},
{
"cell_type": "markdown",
@@ -1325,7 +1327,8 @@
},
"outputs": [],
"source": [
"test_item = ! gcloud storage cat $IMPORT_FILE | head -n1\n", "\n",
"test_item = ! gsutil cat $IMPORT_FILE | head -n1\n",
"\n",
"cols = str(test_item[0]).split(\",\")\n",
"test_item = cols[0]\n",
"test_label = cols[1:]\n",
@@ -1530,7 +1533,8 @@
" print(e)\n",
"\n",
"if delete_bucket and \"BUCKET_NAME\" in globals():\n",
" ! gcloud storage rm --recursive $BUCKET_NAME" ]
" ! gsutil rm -r $BUCKET_NAME"
]
}
],
"metadata": {
@@ -39,7 +39,7 @@
" </td>\n",
" <td>\n",
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/gapic/automl/showcase_automl_text_sentiment_analysis_batch.ipynb\">\n",
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
" View on GitHub\n",
" </a>\n",
" </td>\n",
@@ -421,7 +421,8 @@
},
"outputs": [],
"source": [
"! gcloud storage buckets create --location $REGION $BUCKET_NAME" ]
"! gsutil mb -l $REGION $BUCKET_NAME"
]
},
{
"cell_type": "markdown",
@@ -440,7 +441,8 @@
},
"outputs": [],
"source": [
"! gcloud storage ls --all-versions --long $BUCKET_NAME" ]
"! gsutil ls -al $BUCKET_NAME"
]
},
{
"cell_type": "markdown",
@@ -881,10 +883,12 @@
"else:\n",
" FILE = IMPORT_FILE\n",
"\n",
"count = ! gcloud storage cat $FILE | wc -l\n", "print(\"Number of Examples\", int(count[0]))\n",
"count = ! gsutil cat $FILE | wc -l\n",
"print(\"Number of Examples\", int(count[0]))\n",
"\n",
"print(\"First 10 rows\")\n",
"! gcloud storage cat $FILE | head" ]
"! gsutil cat $FILE | head"
]
},
{
"cell_type": "markdown",
@@ -1312,7 +1316,8 @@
},
"outputs": [],
"source": [
"test_items = ! gcloud storage cat $IMPORT_FILE | head -n2\n", "\n",
"test_items = ! gsutil cat $IMPORT_FILE | head -n2\n",
"\n",
"if len(test_items[0]) == 4:\n",
" _, test_item_1, test_label_1, _ = str(test_items[0]).split(\",\")\n",
" _, test_item_2, test_label_2, _ = str(test_items[1]).split(\",\")\n",
@@ -1370,7 +1375,8 @@
" f.write(json.dumps(data) + \"\\n\")\n",
"\n",
"print(gcs_input_uri)\n",
"! gcloud storage cat $gcs_input_uri" ]
"! gsutil cat $gcs_input_uri"
]
},
{
"cell_type": "markdown",
@@ -1637,7 +1643,8 @@
"source": [
"def get_latest_predictions(gcs_out_dir):\n",
" \"\"\" Get the latest prediction subfolder using the timestamp in the subfolder name\"\"\"\n",
" folders = !gcloud storage ls $gcs_out_dir\n", " latest = \"\"\n",
" folders = !gsutil ls $gcs_out_dir\n",
" latest = \"\"\n",
" for folder in folders:\n",
" subfolder = folder.split(\"/\")[-2]\n",
" if subfolder.startswith(\"prediction-\"):\n",
@@ -1654,8 +1661,10 @@
" raise Exception(\"Batch Job Failed\")\n",
" else:\n",
" folder = get_latest_predictions(predictions)\n",
" ! gcloud storage ls $folder/prediction*.jsonl\n", "\n",
" ! gcloud storage cat $folder/prediction*.jsonl\n", " break\n",
" ! gsutil ls $folder/prediction*.jsonl\n",
"\n",
" ! gsutil cat $folder/prediction*.jsonl\n",
" break\n",
" time.sleep(60)"
]
},
@@ -1749,7 +1758,8 @@
" print(e)\n",
"\n",
"if delete_bucket and \"BUCKET_NAME\" in globals():\n",
" ! gcloud storage rm --recursive $BUCKET_NAME" ]
" ! gsutil rm -r $BUCKET_NAME"
]
}
],
"metadata": {
@@ -39,7 +39,7 @@
" </td>\n",
" <td>\n",
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/gapic/automl/showcase_automl_text_sentiment_analysis_online.ipynb\">\n",
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
" View on GitHub\n",
" </a>\n",
" </td>\n",
@@ -718,10 +718,12 @@
"else:\n",
" FILE = IMPORT_FILE\n",
"\n",
"count = ! gcloud storage cat $FILE | wc -l\n", "print(\"Number of Examples\", int(count[0]))\n",
"count = ! gsutil cat $FILE | wc -l\n",
"print(\"Number of Examples\", int(count[0]))\n",
"\n",
"print(\"First 10 rows\")\n",
"! gcloud storage cat $FILE | head" ]
"! gsutil cat $FILE | head"
]
},
{
"cell_type": "markdown",
@@ -1327,7 +1329,8 @@
},
"outputs": [],
"source": [
"test_item = ! gcloud storage cat $IMPORT_FILE | head -n1\n", "if len(test_item[0]) == 3:\n",
"test_item = ! gsutil cat $IMPORT_FILE | head -n1\n",
"if len(test_item[0]) == 3:\n",
" _, test_item, test_label, max = str(test_item[0]).split(\",\")\n",
"else:\n",
" test_item, test_label, max = str(test_item[0]).split(\",\")\n",
@@ -1531,7 +1534,8 @@
" print(e)\n",
"\n",
"if delete_bucket and \"BUCKET_NAME\" in globals():\n",
" ! gcloud storage rm --recursive $BUCKET_NAME" ]
" ! gsutil rm -r $BUCKET_NAME"
]
}
],
"metadata": {
@@ -39,7 +39,7 @@
" </td>\n",
" <td>\n",
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/gapic/automl/showcase_automl_video_action_recognition_batch.ipynb\">\n",
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
" View on GitHub\n",
" </a>\n",
" </td>\n",
@@ -420,7 +420,8 @@
},
"outputs": [],
"source": [
"! gcloud storage buckets create --location=$REGION $BUCKET_NAME" ]
"! gsutil mb -l $REGION $BUCKET_NAME"
]
},
{
"cell_type": "markdown",
@@ -439,7 +440,8 @@
},
"outputs": [],
"source": [
"! gcloud storage ls --all-versions --long $BUCKET_NAME" ]
"! gsutil ls -al $BUCKET_NAME"
]
},
{
"cell_type": "markdown",
@@ -886,10 +888,12 @@
"else:\n",
" FILE = IMPORT_FILE\n",
"\n",
"count = ! gcloud storage cat $FILE | wc -l\n", "print(\"Number of Examples\", int(count[0]))\n",
"count = ! gsutil cat $FILE | wc -l\n",
"print(\"Number of Examples\", int(count[0]))\n",
"\n",
"print(\"First 10 rows\")\n",
"! gcloud storage cat $FILE | head" ]
"! gsutil cat $FILE | head"
]
},
{
"cell_type": "markdown",
@@ -1306,7 +1310,8 @@
"import json\n",
"\n",
"import_file = IMPORT_FILES[0]\n",
"test_items = ! gcloud storage cat $import_file | head -n2\n", "\n",
"test_items = ! gsutil cat $import_file | head -n2\n",
"\n",
"cols = str(test_items[0]).split(',')\n",
"test_item_1 = str(cols[0])\n",
"test_label_1 = str(cols[-1])\n",
@@ -1355,7 +1360,8 @@
" f.write(json.dumps(data) + '\\n')\n",
"\n",
"print(gcs_input_uri)\n",
"! gcloud storage cat $gcs_input_uri" ]
"! gsutil cat $gcs_input_uri"
]
},
{
"cell_type": "markdown",
@@ -1631,7 +1637,8 @@
"source": [
"def get_latest_predictions(gcs_out_dir):\n",
" ''' Get the latest prediction subfolder using the timestamp in the subfolder name'''\n",
" folders = !gcloud storage ls $gcs_out_dir\n", " latest = \"\"\n",
" folders = !gsutil ls $gcs_out_dir\n",
" latest = \"\"\n",
" for folder in folders:\n",
" subfolder = folder.split('/')[-2]\n",
" if subfolder.startswith('prediction-'):\n",
@@ -1648,8 +1655,10 @@
" raise Exception(\"Batch Job Failed\")\n",
" else:\n",
" folder = get_latest_predictions(predictions)\n",
" ! gcloud storage ls $folder/prediction*.jsonl\n", "\n",
" ! gcloud storage cat $folder/prediction*.jsonl\n", " break\n",
" ! gsutil ls $folder/prediction*.jsonl\n",
"\n",
" ! gsutil cat $folder/prediction*.jsonl\n",
" break\n",
" time.sleep(60)"
]
},
@@ -1743,7 +1752,8 @@
" print(e)\n",
"\n",
"if delete_bucket and 'BUCKET_NAME' in globals():\n",
" ! gcloud storage rm --recursive $BUCKET_NAME" ]
" ! gsutil rm -r $BUCKET_NAME"
]
}
],
"metadata": {
@@ -39,7 +39,7 @@
" </td>\n",
" <td>\n",
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/gapic/automl/showcase_automl_video_classification_batch.ipynb\">\n",
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
" View on GitHub\n",
" </a>\n",
" </td>\n",
@@ -420,7 +420,8 @@
},
"outputs": [],
"source": [
"! gcloud storage buckets create --location $REGION $BUCKET_NAME" ]
"! gsutil mb -l $REGION $BUCKET_NAME"
]
},
{
"cell_type": "markdown",
@@ -439,7 +440,8 @@
},
"outputs": [],
"source": [
"! gcloud storage ls --all-versions --long $BUCKET_NAME" ]
"! gsutil ls -al $BUCKET_NAME"
]
},
{
"cell_type": "markdown",
@@ -883,10 +885,12 @@
"else:\n",
" FILE = IMPORT_FILE\n",
"\n",
"count = ! gcloud storage cat $FILE | wc -l\n", "print(\"Number of Examples\", int(count[0]))\n",
"count = ! gsutil cat $FILE | wc -l\n",
"print(\"Number of Examples\", int(count[0]))\n",
"\n",
"print(\"First 10 rows\")\n",
"! gcloud storage cat $FILE | head" ]
"! gsutil cat $FILE | head"
]
},
{
"cell_type": "markdown",
@@ -1297,7 +1301,8 @@
},
"outputs": [],
"source": [
"test_items = ! gcloud storage cat $IMPORT_FILE | head -n2\n", "\n",
"test_items = ! gsutil cat $IMPORT_FILE | head -n2\n",
"\n",
"if len(test_items[0]) == 5:\n",
" _, test_item_1, test_label_1, _, _ = str(test_items[0]).split(',')\n",
" _, test_item_2, test_label_2, _, _ = str(test_items[1]).split(',')\n",
@@ -1346,7 +1351,8 @@
" f.write(json.dumps(data) + '\\n')\n",
"\n",
"print(gcs_input_uri)\n",
"! gcloud storage cat $gcs_input_uri" ]
"! gsutil cat $gcs_input_uri"
]
},
{
"cell_type": "markdown",
@@ -1624,7 +1630,8 @@
"source": [
"def get_latest_predictions(gcs_out_dir):\n",
" ''' Get the latest prediction subfolder using the timestamp in the subfolder name'''\n",
" folders = !gcloud storage ls $gcs_out_dir\n", " latest = \"\"\n",
" folders = !gsutil ls $gcs_out_dir\n",
" latest = \"\"\n",
" for folder in folders:\n",
" subfolder = folder.split('/')[-2]\n",
" if subfolder.startswith('prediction-'):\n",
@@ -1641,8 +1648,10 @@
" raise Exception(\"Batch Job Failed\")\n",
" else:\n",
" folder = get_latest_predictions(predictions)\n",
" ! gcloud storage ls $folder/prediction*.jsonl\n", "\n",
"! gcloud storage cat $folder/prediction*.jsonl\n", " break\n",
" ! gsutil ls $folder/prediction*.jsonl\n",
"\n",
" ! gsutil cat $folder/prediction*.jsonl\n",
" break\n",
" time.sleep(60)"
]
},
@@ -1736,7 +1745,8 @@
" print(e)\n",
"\n",
"if delete_bucket and 'BUCKET_NAME' in globals():\n",
" ! gcloud storage rm --recursive $BUCKET_NAME" ]
" ! gsutil rm -r $BUCKET_NAME"
]
}
],
"metadata": {
@@ -39,7 +39,7 @@
" </td>\n",
" <td>\n",
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/gapic/automl/showcase_automl_video_object_tracking_batch.ipynb\">\n",
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
" View on GitHub\n",
" </a>\n",
" </td>\n",
@@ -420,7 +420,8 @@
},
"outputs": [],
"source": [
"! gcloud storage buckets create --location $REGION $BUCKET_NAME" ]
"! gsutil mb -l $REGION $BUCKET_NAME"
]
},
{
"cell_type": "markdown",
@@ -439,7 +440,8 @@
},
"outputs": [],
"source": [
"! gcloud storage ls --all-versions --long $BUCKET_NAME" ]
"! gsutil ls -al $BUCKET_NAME"
]
},
{
"cell_type": "markdown",
@@ -888,10 +890,12 @@
"else:\n",
" FILE = IMPORT_FILE\n",
"\n",
"count = ! gcloud storage cat $FILE | wc -l\n", "print(\"Number of Examples\", int(count[0]))\n",
"count = ! gsutil cat $FILE | wc -l\n",
"print(\"Number of Examples\", int(count[0]))\n",
"\n",
"print(\"First 10 rows\")\n",
"! gcloud storage cat $FILE | head" ]
"! gsutil cat $FILE | head"
]
},
{
"cell_type": "markdown",
@@ -1305,7 +1309,8 @@
},
"outputs": [],
"source": [
"test_items = ! gcloud storage cat $IMPORT_FILE | head -n2\n", "\n",
"test_items = ! gsutil cat $IMPORT_FILE | head -n2\n",
"\n",
"cols_1 = test_items[0].split(',')\n",
"cols_2 = test_items[1].split(',')\n",
"if len(cols_1) > 12:\n",
@@ -1360,7 +1365,8 @@
" f.write(json.dumps(data) + '\\n')\n",
"\n",
"print(gcs_input_uri)\n",
"! gcloud storage cat $gcs_input_uri" ]
"! gsutil cat $gcs_input_uri"
]
},
{
"cell_type": "markdown",
@@ -1632,7 +1638,8 @@
"source": [
"def get_latest_predictions(gcs_out_dir):\n",
" ''' Get the latest prediction subfolder using the timestamp in the subfolder name'''\n",
" folders = !gcloud storage ls $gcs_out_dir\n", " latest = \"\"\n",
" folders = !gsutil ls $gcs_out_dir\n",
" latest = \"\"\n",
" for folder in folders:\n",
" subfolder = folder.split('/')[-2]\n",
" if subfolder.startswith('prediction-'):\n",
@@ -1649,8 +1656,10 @@
" raise Exception(\"Batch Job Failed\")\n",
" else:\n",
" folder = get_latest_predictions(predictions)\n",
" ! gcloud storage ls $folder/prediction*.jsonl\n", "\n",
" ! gcloud storage cat $folder/prediction*.jsonl\n", " break\n",
" ! gsutil ls $folder/prediction*.jsonl\n",
"\n",
" ! gsutil cat $folder/prediction*.jsonl\n",
" break\n",
" time.sleep(60)"
]
},
@@ -1744,7 +1753,8 @@
" print(e)\n",
"\n",
"if delete_bucket and 'BUCKET_NAME' in globals():\n",
" ! gcloud storage rm --recursive $BUCKET_NAME" ]
" ! gsutil rm -r $BUCKET_NAME"
]
}
],
"metadata": {
@@ -39,7 +39,7 @@
" </td>\n",
" <td>\n",
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/gapic/custom/showcase_custom_image_classification_batch.ipynb\">\n",
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
" View on GitHub\n",
" </a>\n",
" </td>\n",
@@ -39,7 +39,7 @@
" </td>\n",
" <td>\n",
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/gapic/custom/showcase_custom_image_classification_batch_explain.ipynb\">\n",
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
" View on GitHub\n",
" </a>\n",
" </td>\n",
@@ -39,7 +39,7 @@
" </td>\n",
" <td>\n",
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/gapic/custom/showcase_custom_image_classification_online.ipynb\">\n",
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
" View on GitHub\n",
" </a>\n",
" </td>\n",
@@ -39,7 +39,7 @@
" </td>\n",
" <td>\n",
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/gapic/custom/showcase_custom_image_classification_online_ab_testing.ipynb\">\n",
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
" View on GitHub\n",
" </a>\n",
" </td>\n",
@@ -39,7 +39,7 @@
" </td>\n",
" <td>\n",
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/gapic/custom/showcase_custom_image_classification_online_container.ipynb\">\n",
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
" View on GitHub\n",
" </a>\n",
" </td>\n",
@@ -39,7 +39,7 @@
" </td>\n",
" <td>\n",
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/gapic/custom/showcase_custom_image_classification_online_explain.ipynb\">\n",
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
" View on GitHub\n",
" </a>\n",
" </td>\n",
@@ -40,7 +40,7 @@
" </td>\n",
" <td>\n",
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/gapic/custom/showcase_custom_image_classification_online_explain_example_based_api.ipynb\">\n",
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
" View on GitHub\n",
" </a>\n",
" </td>\n",
@@ -39,7 +39,7 @@
" </td>\n",
" <td>\n",
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/gapic/custom/showcase_custom_image_classification_online_exported_ds.ipynb\">\n",
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
" View on GitHub\n",
" </a>\n",
" </td>\n",
@@ -39,7 +39,7 @@
" </td>\n",
" <td>\n",
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/gapic/custom/showcase_custom_image_classification_online_pipeline.ipynb\">\n",
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
" View on GitHub\n",
" </a>\n",
" </td>\n",
@@ -39,7 +39,7 @@
" </td>\n",
" <td>\n",
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/gapic/custom/showcase_custom_image_classification_online_raw.ipynb\">\n",
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
" View on GitHub\n",
" </a>\n",
" </td>\n",
@@ -39,7 +39,7 @@
" </td>\n",
" <td>\n",
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/gapic/custom/showcase_custom_image_classification_online_tfserving.ipynb\">\n",
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
" View on GitHub\n",
" </a>\n",
" </td>\n",
@@ -39,7 +39,7 @@
" </td>\n",
" <td>\n",
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/gapic/custom/showcase_custom_image_super_resolution_online_post.ipynb\">\n",
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
" View on GitHub\n",
" </a>\n",
" </td>\n",
@@ -39,7 +39,7 @@
" </td>\n",
" <td>\n",
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/gapic/custom/showcase_custom_tabular_classification_online_exported_ds.ipynb\">\n",
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
" View on GitHub\n",
" </a>\n",
" </td>\n",
@@ -39,7 +39,7 @@
" </td>\n",
" <td>\n",
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/gapic/custom/showcase_custom_tabular_regression_batch.ipynb\">\n",
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
" View on GitHub\n",
" </a>\n",
" </td>\n",
@@ -39,7 +39,7 @@
" </td>\n",
" <td>\n",
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/gapic/custom/showcase_custom_tabular_regression_batch_explain.ipynb\">\n",
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
" View on GitHub\n",
" </a>\n",
" </td>\n",
@@ -39,7 +39,7 @@
" </td>\n",
" <td>\n",
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/gapic/custom/showcase_custom_tabular_regression_online.ipynb\">\n",
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
" View on GitHub\n",
" </a>\n",
" </td>\n",
@@ -39,7 +39,7 @@
" </td>\n",
" <td>\n",
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/gapic/custom/showcase_custom_tabular_regression_online_container.ipynb\">\n",
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
" View on GitHub\n",
" </a>\n",
" </td>\n",
@@ -39,7 +39,7 @@
" </td>\n",
" <td>\n",
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/gapic/custom/showcase_custom_tabular_regression_online_explain.ipynb\">\n",
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
" View on GitHub\n",
" </a>\n",
" </td>\n",
@@ -39,7 +39,7 @@
" </td>\n",
" <td>\n",
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/gapic/custom/showcase_custom_tabular_regression_online_pipeline.ipynb\">\n",
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
" View on GitHub\n",
" </a>\n",
" </td>\n",
@@ -39,7 +39,7 @@
" </td>\n",
" <td>\n",
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/gapic/custom/showcase_custom_tabular_regression_online_tfserving.ipynb\">\n",
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
" View on GitHub\n",
" </a>\n",
" </td>\n",
@@ -39,7 +39,7 @@
" </td>\n",
" <td>\n",
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/gapic/custom/showcase_custom_text_binary_classification_batch.ipynb\">\n",
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
" View on GitHub\n",
" </a>\n",
" </td>\n",
@@ -39,7 +39,7 @@
" </td>\n",
" <td>\n",
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/gapic/custom/showcase_custom_text_binary_classification_online.ipynb\">\n",
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
" View on GitHub\n",
" </a>\n",
" </td>\n",
@@ -39,7 +39,7 @@
" </td>\n",
" <td>\n",
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/gapic/custom/showcase_custom_text_binary_classification_online_container.ipynb\">\n",
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
" View on GitHub\n",
" </a>\n",
" </td>\n",
@@ -39,7 +39,7 @@
" </td>\n",
" <td>\n",
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/gapic/custom/showcase_custom_text_binary_classification_online_pipeline.ipynb\">\n",
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
" View on GitHub\n",
" </a>\n",
" </td>\n",
@@ -39,7 +39,7 @@
" </td>\n",
" <td>\n",
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/gapic/custom/showcase_custom_text_binary_classification_online_tfserving.ipynb\">\n",
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
" View on GitHub\n",
" </a>\n",
" </td>\n",
@@ -39,7 +39,7 @@
" </td>\n",
" <td>\n",
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/gapic/custom/showcase_custom_text_classification_online_exported_ds.ipynb\">\n",
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
" View on GitHub\n",
" </a>\n",
" </td>\n",
@@ -426,7 +426,8 @@
},
"outputs": [],
"source": [
"! gcloud storage buckets create --location=$REGION $BUCKET_NAME" ]
"! gsutil mb -l $REGION $BUCKET_NAME"
]
},
{
"cell_type": "markdown",
@@ -445,7 +446,8 @@
},
"outputs": [],
"source": [
"! gcloud storage ls --all-versions --long $BUCKET_NAME" ]
"! gsutil ls -al $BUCKET_NAME"
]
},
{
"cell_type": "markdown",
@@ -481,6 +483,9 @@
"import time\n",
"\n",
"from google.cloud.aiplatform import gapic as aip\n",
"from google.protobuf import json_format\n",
"from google.protobuf.json_format import MessageToJson, ParseDict\n",
"from google.protobuf.struct_pb2 import Struct, Value"
]
},
{
@@ -986,10 +991,12 @@
"else:\n",
" FILE = IMPORT_FILE\n",
"\n",
"count = ! gcloud storage cat $FILE | wc -l\n", "print(\"Number of Examples\", int(count[0]))\n",
"count = ! gsutil cat $FILE | wc -l\n",
"print(\"Number of Examples\", int(count[0]))\n",
"\n",
"print(\"First 10 rows\")\n",
"! gcloud storage cat $FILE | head" ]
"! gsutil cat $FILE | head"
]
},
{
"cell_type": "markdown",
@@ -1146,7 +1153,8 @@
"source": [
"jsonl_index = result.exported_files[0]\n",
"\n",
"! gcloud storage cat $jsonl_index | head" ]
"! gsutil cat $jsonl_index | head"
]
},
{
"cell_type": "markdown",
@@ -1278,7 +1286,8 @@
" print(e)\n",
"\n",
"if delete_bucket and \"BUCKET_NAME\" in globals():\n",
" ! gcloud storage rm --recursive $BUCKET_NAME" ]
" ! gsutil rm -r $BUCKET_NAME"
]
}
],
"metadata": {
@@ -39,7 +39,7 @@
" </td>\n",
" <td>\n",
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/gapic/custom/showcase_hyperparmeter_tuning_image_classification.ipynb\">\n",
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
" View on GitHub\n",
" </a>\n",
" </td>\n",
@@ -419,7 +419,8 @@
},
"outputs": [],
"source": [
"! gcloud storage buckets create --location=$REGION $BUCKET_NAME" ]
"! gsutil mb -l $REGION $BUCKET_NAME"
]
},
{
"cell_type": "markdown",
@@ -438,7 +439,8 @@
},
"outputs": [],
"source": [
"! gcloud storage ls --all-versions --long $BUCKET_NAME" ]
"! gsutil ls -al $BUCKET_NAME"
]
},
{
"cell_type": "markdown",
@@ -1117,7 +1119,8 @@
"! rm -f custom.tar custom.tar.gz\n",
"! tar cvf custom.tar custom\n",
"! gzip custom.tar\n",
"! gcloud storage cp custom.tar.gz $BUCKET_NAME/trainer_cifar10.tar.gz" ]
"! gsutil cp custom.tar.gz $BUCKET_NAME/trainer_cifar10.tar.gz"
]
},
{
"cell_type": "markdown",
@@ -1614,7 +1617,8 @@
"! rm -f custom.tar custom.tar.gz\n",
"! tar cvf custom.tar custom\n",
"! gzip custom.tar\n",
"! gcloud storage cp custom.tar.gz $BUCKET_NAME/trainer_cifar10.tar.gz" ]
"! gsutil cp custom.tar.gz $BUCKET_NAME/trainer_cifar10.tar.gz"
]
},
{
"cell_type": "markdown",
@@ -2058,7 +2062,8 @@
" print(e)\n",
"\n",
"if delete_bucket and \"BUCKET_NAME\" in globals():\n",
" ! gcloud storage rm --recursive $BUCKET_NAME" ]
" ! gsutil rm -r $BUCKET_NAME"
]
}
],
"metadata": {
@@ -39,7 +39,7 @@
" </td>\n",
" <td>\n",
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/gapic/custom/showcase_hyperparmeter_tuning_tabular_regression.ipynb\">\n",
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
" View on GitHub\n",
" </a>\n",
" </td>\n",
@@ -419,7 +419,8 @@
},
"outputs": [],
"source": [
"! gcloud storage buckets create --location $REGION $BUCKET_NAME" ]
"! gsutil mb -l $REGION $BUCKET_NAME"
]
},
{
"cell_type": "markdown",
@@ -438,7 +439,8 @@
},
"outputs": [],
"source": [
"! gcloud storage ls --all-versions --long $BUCKET_NAME" ]
"! gsutil ls -al $BUCKET_NAME"
]
},
{
"cell_type": "markdown",
@@ -1114,7 +1116,8 @@
"! rm -f custom.tar custom.tar.gz\n",
"! tar cvf custom.tar custom\n",
"! gzip custom.tar\n",
"! gcloud storage cp custom.tar.gz $BUCKET_NAME/trainer_boston.tar.gz" ]
"! gsutil cp custom.tar.gz $BUCKET_NAME/trainer_boston.tar.gz"
]
},
{
"cell_type": "markdown",
@@ -1623,7 +1626,8 @@
"! rm -f custom.tar custom.tar.gz\n",
"! tar cvf custom.tar custom\n",
"! gzip custom.tar\n",
"! gcloud storage cp custom.tar.gz $BUCKET_NAME/trainer_boston.tar.gz" ]
"! gsutil cp custom.tar.gz $BUCKET_NAME/trainer_boston.tar.gz"
]
},
{
"cell_type": "markdown",
@@ -2081,7 +2085,8 @@
" print(e)\n",
"\n",
"if delete_bucket and \"BUCKET_NAME\" in globals():\n",
" ! gcloud storage rm --recursive $BUCKET_NAME" ]
" ! gsutil rm -r $BUCKET_NAME"
]
}
],
"metadata": {
@@ -39,7 +39,7 @@
" </td>\n",
" <td>\n",
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/gapic/custom/showcase_hyperparmeter_tuning_text_binary_classification.ipynb\">\n",
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
" View on GitHub\n",
" </a>\n",
" </td>\n",
@@ -419,7 +419,8 @@
},
"outputs": [],
"source": [
"! gcloud storage buckets create --location=$REGION $BUCKET_NAME" ]
"! gsutil mb -l $REGION $BUCKET_NAME"
]
},
{
"cell_type": "markdown",
@@ -438,7 +439,8 @@
},
"outputs": [],
"source": [
"! gcloud storage ls --all-versions --long $BUCKET_NAME" ]
"! gsutil ls -al $BUCKET_NAME"
]
},
{
"cell_type": "markdown",
@@ -1114,7 +1116,8 @@
"! rm -f custom.tar custom.tar.gz\n",
"! tar cvf custom.tar custom\n",
"! gzip custom.tar\n",
"! gcloud storage cp custom.tar.gz $BUCKET_NAME/trainer_imdb.tar.gz" ]
"! gsutil cp custom.tar.gz $BUCKET_NAME/trainer_imdb.tar.gz"
]
},
{
"cell_type": "markdown",
@@ -1615,7 +1618,8 @@
"! rm -f custom.tar custom.tar.gz\n",
"! tar cvf custom.tar custom\n",
"! gzip custom.tar\n",
"! gcloud storage cp custom.tar.gz $BUCKET_NAME/trainer_imdb.tar.gz" ]
"! gsutil cp custom.tar.gz $BUCKET_NAME/trainer_imdb.tar.gz"
]
},
{
"cell_type": "markdown",
@@ -2056,7 +2060,8 @@
" print(e)\n",
"\n",
"if delete_bucket and \"BUCKET_NAME\" in globals():\n",
" ! gcloud storage rm --recursive $BUCKET_NAME" ]
" ! gsutil rm -r $BUCKET_NAME"
]
}
],
"metadata": {

Some files were not shown because too many files have changed in this diff Show More