mirror of
https://github.com/GoogleCloudPlatform/vertex-ai-samples.git
synced 2026-09-26 14:42:04 +00:00
Compare commits
221
Commits
cl41
...
href_updates
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
3b22758103 | ||
|
|
09f5ec1361 | ||
|
|
d7d3b76487 | ||
|
|
0417d8b9c4 | ||
|
|
7750e83fbb | ||
|
|
649800e646 | ||
|
|
42b35056fa | ||
|
|
24974eda95 | ||
|
|
ad41377783 | ||
|
|
36dea3ca01 | ||
|
|
b75b2ea4d7 | ||
|
|
28f7fc4445 | ||
|
|
bf2c1226fd | ||
|
|
2990c53292 | ||
|
|
531d9cfee0 | ||
|
|
ff18ec7af5 | ||
|
|
008eb409ef | ||
|
|
b2dba4b568 | ||
|
|
665547f790 | ||
|
|
5078c44eb8 | ||
|
|
da6e46531e | ||
|
|
0a4091a3b1 | ||
|
|
5afa83dd25 | ||
|
|
8cab85d6ad | ||
|
|
a7f3940635 | ||
|
|
5bb1a75a48 | ||
|
|
8fe4985aa8 | ||
|
|
996b6534d9 | ||
|
|
1061ae5348 | ||
|
|
4fac3a630f | ||
|
|
1f2db2903c | ||
|
|
912a52de70 | ||
|
|
5e29090e86 | ||
|
|
cd8fcd1839 | ||
|
|
e51075ec4b | ||
|
|
9709c0dddb | ||
|
|
a21ae41762 | ||
|
|
ff5ff7b609 | ||
|
|
23748f443e | ||
|
|
fc6b2167de | ||
|
|
e79a45358c | ||
|
|
20d19fb11c | ||
|
|
9c9f7a6e2a | ||
|
|
ede41c2115 | ||
|
|
ca53786c04 | ||
|
|
419f8310c9 | ||
|
|
996b690e03 | ||
|
|
5b9d04d63a | ||
|
|
9ed3c2f83d | ||
|
|
5fc93c8bdb | ||
|
|
3cf46226e9 | ||
|
|
439f6a0cae | ||
|
|
993898bb71 | ||
|
|
9e9e639375 | ||
|
|
633cf6a799 | ||
|
|
8b618bc455 | ||
|
|
6bbe3bcfe0 | ||
|
|
814827ac19 | ||
|
|
7a613785b9 | ||
|
|
100243e90a | ||
|
|
6424515b03 | ||
|
|
8d22b221b4 | ||
|
|
2a8ad7cdbb | ||
|
|
3239b301f2 | ||
|
|
acb10d14b8 | ||
|
|
778d145970 | ||
|
|
4d00356f4b | ||
|
|
0654305994 | ||
|
|
c53f392c5a | ||
|
|
f4b56e92ae | ||
|
|
45ec1cf18a | ||
|
|
f6c8bcf937 | ||
|
|
9e590d5a9f | ||
|
|
46e0ea4f1c | ||
|
|
300fce6b9f | ||
|
|
52e3066c38 | ||
|
|
27ebf52198 | ||
|
|
ca7d4e153e | ||
|
|
5b6c766629 | ||
|
|
5efa51206f | ||
|
|
e604a4d43e | ||
|
|
23af5373ec | ||
|
|
7577c0b1fc | ||
|
|
b648f9e73b | ||
|
|
966bbc49a7 | ||
|
|
f5d341ae45 | ||
|
|
70770a50c7 | ||
|
|
f181c39cbf | ||
|
|
a5637f87f2 | ||
|
|
0103299084 | ||
|
|
1867536d76 | ||
|
|
0edae683e7 | ||
|
|
8471b5cb6f | ||
|
|
87f540ac53 | ||
|
|
babeba9f02 | ||
|
|
85c649dd26 | ||
|
|
b7135ae1f0 | ||
|
|
2f5119a266 | ||
|
|
23e64ca76f | ||
|
|
0ba5a62cc9 | ||
|
|
0be2c6fd0c | ||
|
|
cef4928c49 | ||
|
|
9bb8107110 | ||
|
|
b075990d88 | ||
|
|
6a83c4c695 | ||
|
|
820c0f8db4 | ||
|
|
4ab197a4ba | ||
|
|
2a5877fbd1 | ||
|
|
090e1d9fee | ||
|
|
2e049d4830 | ||
|
|
e3320d2126 | ||
|
|
5fc0e03ca3 | ||
|
|
ff2a16237d | ||
|
|
d26f081642 | ||
|
|
8a0a39176c | ||
|
|
646532ea69 | ||
|
|
9e96a3da67 | ||
|
|
db34e1fbd5 | ||
|
|
82308acbac | ||
|
|
ee0ba75d1e | ||
|
|
d61aedc721 | ||
|
|
19f7f94af5 | ||
|
|
4e5ce9b226 | ||
|
|
99938244f4 | ||
|
|
0cc7be4a6a | ||
|
|
b6bde41850 | ||
|
|
52444a0933 | ||
|
|
bab9c398fd | ||
|
|
447affcc93 | ||
|
|
6132c37be0 | ||
|
|
8d7f59aeec | ||
|
|
bd327ad424 | ||
|
|
3b1fbdb382 | ||
|
|
954043a729 | ||
|
|
16ef9ee80e | ||
|
|
79301b4a4d | ||
|
|
0b38d02e6f | ||
|
|
c52ff25ba4 | ||
|
|
424400bace | ||
|
|
b1dfac2043 | ||
|
|
b81ffcddab | ||
|
|
21d8f144aa | ||
|
|
065a674305 | ||
|
|
571d498d08 | ||
|
|
e936882123 | ||
|
|
f754f99052 | ||
|
|
aa5523a5e9 | ||
|
|
81393ede1a | ||
|
|
5a1c0222da | ||
|
|
1f9326bd56 | ||
|
|
a94cae2e79 | ||
|
|
cb861713c8 | ||
|
|
3a55087789 | ||
|
|
d359b21f3e | ||
|
|
c7d4123b25 | ||
|
|
07a8bb2d0c | ||
|
|
f6b6f365b6 | ||
|
|
fec825f9e5 | ||
|
|
1e9bf72097 | ||
|
|
3fa1cf99ec | ||
|
|
2bcaf8abde | ||
|
|
759495a1f8 | ||
|
|
9925e62c4b | ||
|
|
40ade71b35 | ||
|
|
65173071a5 | ||
|
|
a514bb51c2 | ||
|
|
a6dc1b0f6d | ||
|
|
75236998c7 | ||
|
|
9c8a7808bf | ||
|
|
86ce1576d2 | ||
|
|
c2b743bfeb | ||
|
|
b04575d746 | ||
|
|
b90d16885e | ||
|
|
828f1a26fa | ||
|
|
915e5edf0a | ||
|
|
5d4c1be285 | ||
|
|
275bfb8f69 | ||
|
|
93cb0ca3d9 | ||
|
|
41d60d052d | ||
|
|
d872cdcf7e | ||
|
|
ada5e4a854 | ||
|
|
ecb32b099d | ||
|
|
f8d09e8e9b | ||
|
|
007df88fba | ||
|
|
c70f3ef9a3 | ||
|
|
505e101452 | ||
|
|
b77d51b58b | ||
|
|
deaa1ccf2b | ||
|
|
f6e38860aa | ||
|
|
4937e382b1 | ||
|
|
0fe2770947 | ||
|
|
80d7ee67d6 | ||
|
|
f17e2d6c8d | ||
|
|
ae1cd0ed08 | ||
|
|
a68b491edc | ||
|
|
d39bed012a | ||
|
|
f8c93de1c4 | ||
|
|
0b5d97aa3a | ||
|
|
87f0785ed1 | ||
|
|
09c8ea616b | ||
|
|
6c11241cf7 | ||
|
|
9b7dc2e4fd | ||
|
|
9b3d43b1a6 | ||
|
|
57ee4e2eab | ||
|
|
4d736d7992 | ||
|
|
69e92a650d | ||
|
|
4ed979eec4 | ||
|
|
0344de8090 | ||
|
|
83eca0f0cc | ||
|
|
4619b272fa | ||
|
|
091fa33c01 | ||
|
|
42a05c2b35 | ||
|
|
8ac32fa42f | ||
|
|
0303057f11 | ||
|
|
7ae13b346a | ||
|
|
44390cbd99 | ||
|
|
58dab0b1bb | ||
|
|
8d36834fcd | ||
|
|
31ca3e36f4 | ||
|
|
a72d7dc49f | ||
|
|
9efbd48233 |
@@ -365,7 +365,7 @@ def process_and_execute_notebook(
|
||||
# Use gcloud to get tail
|
||||
try:
|
||||
result.error_message = subprocess.check_output(
|
||||
["gsutil", "cat", "-r", "-1000", log_file_uri], encoding="UTF-8"
|
||||
["gcloud", "storage", "cat", "--range", "-1000", log_file_uri], encoding="UTF-8"
|
||||
)
|
||||
except Exception as error:
|
||||
result.error_message = str(error)
|
||||
|
||||
@@ -56,8 +56,8 @@ def execute_notebook(
|
||||
print("\n=== DOWNLOAD EXECUTED NOTEBOOK ===\n")
|
||||
print(f"Please debug the executed notebook by downloading the executed notebook:")
|
||||
|
||||
print("Option 1. Using gsutil. Run the following command in your terminal.")
|
||||
print(f'\tgsutil cp "{output_file_or_uri}" .')
|
||||
print("Option 1. Using gcloud storage. Run the following command in your terminal.")
|
||||
print(f'\tgcloud storage cp "{output_file_or_uri}" .')
|
||||
|
||||
print("Option 2. Using this link.")
|
||||
print(f"\thttps://storage.googleapis.com/{output_file_or_uri[5:]}")
|
||||
|
||||
@@ -108,7 +108,7 @@ class VertexAIInstallProprocessor(Preprocessor):
|
||||
if "google-cloud-aiplatform" not in content:
|
||||
return content
|
||||
return (
|
||||
f"gsutil cp {self.vertex_ai_wheel} google-cloud-aiplatform.whl\n" +
|
||||
f"gcloud storage cp {self.vertex_ai_wheel} google-cloud-aiplatform.whl\n" +
|
||||
content.replace("google-cloud-aiplatform\n", "google-cloud-aiplatform.whl\n")
|
||||
.replace("google-cloud-aiplatform ", "google-cloud-aiplatform.whl ")
|
||||
)
|
||||
|
||||
@@ -15,7 +15,7 @@ def download_file(bucket_name: str, blob_name: str, destination_file: str) -> st
|
||||
remote_file_path = "".join(["gs://", "/".join([bucket_name, blob_name])])
|
||||
|
||||
subprocess.check_output(
|
||||
["gsutil", "cp", remote_file_path, destination_file], encoding="UTF-8"
|
||||
["gcloud", "storage", "cp", remote_file_path, destination_file], encoding="UTF-8"
|
||||
)
|
||||
|
||||
return destination_file
|
||||
@@ -27,7 +27,7 @@ def upload_file(
|
||||
) -> str:
|
||||
"""Copies a local file to a GCS path"""
|
||||
subprocess.check_output(
|
||||
["gsutil", "cp", local_file_path, remote_file_path], encoding="UTF-8"
|
||||
["gcloud", "storage", "cp", local_file_path, remote_file_path], encoding="UTF-8"
|
||||
)
|
||||
|
||||
return remote_file_path
|
||||
|
||||
@@ -7,11 +7,11 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
uses: actions/setup-python@v6
|
||||
with:
|
||||
python-version: '3.x'
|
||||
python-version: '3.12'
|
||||
- name: Fetch pull request branch
|
||||
uses: actions/checkout@v4
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- name: Fetch base main branch
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
# 2. To lint specific notebooks:
|
||||
# docker run -v ${PWD}:/setup/app gcr.io/python-docs-samples-tests/notebook_linter:latest notebooks/1.ipynb notebooks/2.ipynb
|
||||
|
||||
FROM python:3.13
|
||||
FROM python:3.14
|
||||
|
||||
WORKDIR setup
|
||||
|
||||
|
||||
@@ -2,9 +2,9 @@ git+https://github.com/tensorflow/docs
|
||||
ipython
|
||||
jupyter
|
||||
nbconvert
|
||||
black==25.1.0
|
||||
pyupgrade==3.20.0
|
||||
isort==6.0.1
|
||||
black==26.1.0
|
||||
pyupgrade==3.21.2
|
||||
isort==7.0.0
|
||||
flake8==7.3.0
|
||||
nbqa==1.9.1
|
||||
|
||||
|
||||
+1
-1
@@ -148,7 +148,7 @@ implementation:
|
||||
|
||||
# Downloading the model archive from GCS
|
||||
# TODO: Fix gsutil bugs (requires project ID, has auth issues) and use gsutil instead.
|
||||
# gsutil cp "$model_archive_uri" "$model_archive_local_path"
|
||||
# gcloud storage cp "$model_archive_uri" "$model_archive_local_path"
|
||||
pip install google-cloud-storage
|
||||
python -c '
|
||||
import sys
|
||||
|
||||
@@ -24,12 +24,12 @@ implementation:
|
||||
|
||||
# Checking whether the URI points to a single blob, a directory or a URI pattern
|
||||
# URI points to a blob when that URI does not end with slash and listing that URI only yields the same URI
|
||||
if [[ "$uri" != */ ]] && (gsutil ls "$uri" | grep --fixed-strings --line-regexp "$uri"); then
|
||||
if [[ "$uri" != */ ]] && (gcloud storage ls "$uri" | grep --fixed-strings --line-regexp "$uri"); then
|
||||
mkdir -p "$(dirname "$output_path")"
|
||||
gsutil -m cp -r "$uri" "$output_path"
|
||||
gcloud storage cp --recursive "$uri" "$output_path"
|
||||
else
|
||||
mkdir -p "$output_path" # When source path is a directory, gsutil requires the destination to also be a directory
|
||||
gsutil -m rsync -r "$uri" "$output_path" # gsutil cp has different path handling than Linux cp. It always puts the source directory (name) inside the destination directory. gsutil rsync does not have that problem.
|
||||
gcloud storage rsync --recursive "$uri" "$output_path" # gsutil cp has different path handling than Linux cp. It always puts the source directory (name) inside the destination directory. gsutil rsync does not have that problem.
|
||||
fi
|
||||
- inputValue: GCS path
|
||||
- outputPath: Data
|
||||
|
||||
+1492
-1505
File diff suppressed because it is too large
Load Diff
+1
-1
@@ -1,3 +1,3 @@
|
||||
torch==2.2.0
|
||||
torch==2.8.0
|
||||
torchvision==0.9.1
|
||||
tensorboard==2.5.0
|
||||
+3
-3
@@ -110,7 +110,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! gsutil ls $gcs_output_uri_prefix"
|
||||
"! gcloud storage ls $gcs_output_uri_prefix"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -192,7 +192,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! gsutil cp -r $gcs_output_uri_prefix/model ./model_server/"
|
||||
"! gcloud storage cp --recursive $gcs_output_uri_prefix/model ./model_server/"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -556,7 +556,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! gsutil rm -rf $gcs_output_uri_prefix"
|
||||
"! gcloud storage rm --recursive --continue-on-error $gcs_output_uri_prefix"
|
||||
]
|
||||
},
|
||||
{
|
||||
|
||||
+1
-1
@@ -412,7 +412,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! gsutil ls $gcs_output_uri_prefix"
|
||||
"! gcloud storage ls $gcs_output_uri_prefix"
|
||||
]
|
||||
}
|
||||
],
|
||||
|
||||
+1
-1
@@ -77,4 +77,4 @@ echo "After the job is completed successfully, model files will be saved at $JOB
|
||||
|
||||
# # Verify the model was exported
|
||||
# echo "Verify the model was exported:"
|
||||
# gsutil ls ${JOB_DIR}/
|
||||
# gcloud storage ls ${JOB_DIR}/
|
||||
|
||||
+1
-1
@@ -34,4 +34,4 @@ RUN echo "service_envelope=json\n" "inference_address=http://0.0.0.0:${AIP_H
|
||||
USER model-server
|
||||
|
||||
# run Torchserve HTTP serve to respond to prediction requests
|
||||
CMD ["echo", "AIP_STORAGE_URI=${AIP_STORAGE_URI}", ";", "gsutil", "cp", "-r", "${AIP_STORAGE_URI}/${MODEL_NAME}.mar", "/home/model-server/model-store/", ";", "ls", "-ltr", "/home/model-server/model-store/", ";", "torchserve", "--start", "--ts-config=/home/model-server/config.properties", "--models", "${MODEL_NAME}=${MODEL_NAME}.mar", "--model-store", "/home/model-server/model-store"]
|
||||
CMD ["echo", "AIP_STORAGE_URI=${AIP_STORAGE_URI}", ";", "gcloud", "storage", "cp", "--recursive", "${AIP_STORAGE_URI}/${MODEL_NAME}.mar", "/home/model-server/model-store/", ";", "ls", "-ltr", "/home/model-server/model-store/", ";", "torchserve", "--start", "--ts-config=/home/model-server/config.properties", "--models", "${MODEL_NAME}=${MODEL_NAME}.mar", "--model-store", "/home/model-server/model-store"]
|
||||
|
||||
+1
-1
@@ -67,4 +67,4 @@ echo "After the job is completed successfully, model files will be saved at $JOB
|
||||
|
||||
# # Verify the model was exported
|
||||
# echo "Verify the model was exported:"
|
||||
# gsutil ls ${JOB_DIR}/
|
||||
# gcloud storage ls ${JOB_DIR}/
|
||||
|
||||
+4
-9
@@ -478,8 +478,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! gsutil mb -l $REGION $BUCKET_NAME"
|
||||
]
|
||||
"! gcloud storage buckets create --location $REGION $BUCKET_NAME" ]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
@@ -498,8 +497,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! gsutil ls -al $BUCKET_NAME"
|
||||
]
|
||||
"! gcloud storage ls --all-versions --long $BUCKET_NAME" ]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
@@ -582,8 +580,7 @@
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Download the sample data into your RAW_DATA_PATH\n",
|
||||
"! gsutil cp \"gs://cloud-samples-data/vertex-ai/community-content/tf_agents_bandits_movie_recommendation_with_kfp_and_vertex_sdk/u.data\" $RAW_DATA_PATH"
|
||||
]
|
||||
"! gcloud storage cp \"gs://cloud-samples-data/vertex-ai/community-content/tf_agents_bandits_movie_recommendation_with_kfp_and_vertex_sdk/u.data\" $RAW_DATA_PATH" ]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
@@ -1621,9 +1618,7 @@
|
||||
"! gcloud scheduler jobs delete $SIMULATOR_SCHEDULER_JOB --quiet\n",
|
||||
"\n",
|
||||
"# Delete Cloud Storage objects that were created.\n",
|
||||
"! gsutil -m rm -r $PIPELINE_ROOT\n",
|
||||
"! gsutil -m rm -r $TRAINING_ARTIFACTS_DIR"
|
||||
]
|
||||
"! gcloud storage rm --recursive $PIPELINE_ROOT\n", "! gcloud storage rm --recursive $TRAINING_ARTIFACTS_DIR" ]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
|
||||
+75
-44
@@ -398,6 +398,7 @@
|
||||
"if not IS_GOOGLE_CLOUD_NOTEBOOK:\n",
|
||||
" if \"google.colab\" in sys.modules:\n",
|
||||
" from google.colab import auth as google_auth\n",
|
||||
"\n",
|
||||
" google_auth.authenticate_user()\n",
|
||||
"\n",
|
||||
" # If you are running this notebook locally, replace the string below with the\n",
|
||||
@@ -472,7 +473,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! gsutil mb -l $REGION $BUCKET_NAME"
|
||||
"! gcloud storage buckets create --location $REGION $BUCKET_NAME"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -492,7 +493,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! gsutil ls -al $BUCKET_NAME"
|
||||
"! gcloud storage ls --all-versions --long $BUCKET_NAME"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -565,7 +566,7 @@
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Copy the sample data into your DATA_PATH\n",
|
||||
"! gsutil cp \"gs://cloud-samples-data/vertex-ai/community-content/tf_agents_bandits_movie_recommendation_with_kfp_and_vertex_sdk/u.data\" $DATA_PATH"
|
||||
"! gcloud storage cp \"gs://cloud-samples-data/vertex-ai/community-content/tf_agents_bandits_movie_recommendation_with_kfp_and_vertex_sdk/u.data\" $DATA_PATH"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -579,11 +580,15 @@
|
||||
"# Set hyperparameters.\n",
|
||||
"BATCH_SIZE = 8 # @param {type:\"integer\"} Training and prediction batch size.\n",
|
||||
"TRAINING_LOOPS = 5 # @param {type:\"integer\"} Number of training iterations.\n",
|
||||
"STEPS_PER_LOOP = 2 # @param {type:\"integer\"} Number of driver steps per training iteration.\n",
|
||||
"STEPS_PER_LOOP = (\n",
|
||||
" 2 # @param {type:\"integer\"} Number of driver steps per training iteration.\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"# Set MovieLens simulation environment parameters.\n",
|
||||
"RANK_K = 20 # @param {type:\"integer\"} Rank for matrix factorization in the MovieLens environment; also the observation dimension.\n",
|
||||
"NUM_ACTIONS = 20 # @param {type:\"integer\"} Number of actions (movie items) to choose from.\n",
|
||||
"NUM_ACTIONS = (\n",
|
||||
" 20 # @param {type:\"integer\"} Number of actions (movie items) to choose from.\n",
|
||||
")\n",
|
||||
"PER_ARM = False # Use the non-per-arm version of the MovieLens environment.\n",
|
||||
"\n",
|
||||
"# Set agent parameters.\n",
|
||||
@@ -621,7 +626,8 @@
|
||||
"source": [
|
||||
"# Define RL environment.\n",
|
||||
"env = movielens_py_environment.MovieLensPyEnvironment(\n",
|
||||
" DATA_PATH, RANK_K, BATCH_SIZE, num_movies=NUM_ACTIONS, csv_delimiter=\"\\t\")\n",
|
||||
" DATA_PATH, RANK_K, BATCH_SIZE, num_movies=NUM_ACTIONS, csv_delimiter=\"\\t\"\n",
|
||||
")\n",
|
||||
"environment = tf_py_environment.TFPyEnvironment(env)\n",
|
||||
"\n",
|
||||
"# Define RL agent/algorithm.\n",
|
||||
@@ -631,7 +637,8 @@
|
||||
" tikhonov_weight=TIKHONOV_WEIGHT,\n",
|
||||
" alpha=AGENT_ALPHA,\n",
|
||||
" dtype=tf.float32,\n",
|
||||
" accepts_per_arm_features=PER_ARM)\n",
|
||||
" accepts_per_arm_features=PER_ARM,\n",
|
||||
")\n",
|
||||
"print(\"TimeStep Spec (for each batch):\\n\", agent.time_step_spec, \"\\n\")\n",
|
||||
"print(\"Action Spec (for each batch):\\n\", agent.action_spec, \"\\n\")\n",
|
||||
"print(\"Reward Spec (for each batch):\\n\", environment.reward_spec(), \"\\n\")\n",
|
||||
@@ -639,7 +646,8 @@
|
||||
"# Define RL metric.\n",
|
||||
"optimal_reward_fn = functools.partial(\n",
|
||||
" environment_utilities.compute_optimal_reward_with_movielens_environment,\n",
|
||||
" environment=environment)\n",
|
||||
" environment=environment,\n",
|
||||
")\n",
|
||||
"regret_metric = tf_bandit_metrics.RegretMetric(optimal_reward_fn)\n",
|
||||
"metrics = [regret_metric]"
|
||||
]
|
||||
@@ -704,35 +712,38 @@
|
||||
" if training_data_spec_transformation_fn is None:\n",
|
||||
" data_spec = agent.policy.trajectory_spec\n",
|
||||
" else:\n",
|
||||
" data_spec = training_data_spec_transformation_fn(\n",
|
||||
" agent.policy.trajectory_spec)\n",
|
||||
" replay_buffer = trainer.get_replay_buffer(data_spec, environment.batch_size,\n",
|
||||
" steps_per_loop)\n",
|
||||
" data_spec = training_data_spec_transformation_fn(agent.policy.trajectory_spec)\n",
|
||||
" replay_buffer = trainer.get_replay_buffer(\n",
|
||||
" data_spec, environment.batch_size, steps_per_loop\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
" # `step_metric` records the number of individual rounds of bandit interaction;\n",
|
||||
" # that is, (number of trajectories) * batch_size.\n",
|
||||
" step_metric = tf_metrics.EnvironmentSteps()\n",
|
||||
" metrics = [\n",
|
||||
" tf_metrics.NumberOfEpisodes(),\n",
|
||||
" tf_metrics.AverageEpisodeLengthMetric(batch_size=environment.batch_size)\n",
|
||||
" tf_metrics.AverageEpisodeLengthMetric(batch_size=environment.batch_size),\n",
|
||||
" ]\n",
|
||||
" if additional_metrics:\n",
|
||||
" metrics += additional_metrics\n",
|
||||
"\n",
|
||||
" if isinstance(environment.reward_spec(), dict):\n",
|
||||
" metrics += [tf_metrics.AverageReturnMultiMetric(\n",
|
||||
" reward_spec=environment.reward_spec(),\n",
|
||||
" batch_size=environment.batch_size)]\n",
|
||||
" else:\n",
|
||||
" metrics += [\n",
|
||||
" tf_metrics.AverageReturnMetric(batch_size=environment.batch_size)]\n",
|
||||
" tf_metrics.AverageReturnMultiMetric(\n",
|
||||
" reward_spec=environment.reward_spec(), batch_size=environment.batch_size\n",
|
||||
" )\n",
|
||||
" ]\n",
|
||||
" else:\n",
|
||||
" metrics += [tf_metrics.AverageReturnMetric(batch_size=environment.batch_size)]\n",
|
||||
"\n",
|
||||
" # Store intermediate metric results, indexed by metric names.\n",
|
||||
" metric_results = defaultdict(list)\n",
|
||||
"\n",
|
||||
" if training_data_spec_transformation_fn is not None:\n",
|
||||
" def add_batch_fn(data): return replay_buffer.add_batch(training_data_spec_transformation_fn(data)) \n",
|
||||
" \n",
|
||||
"\n",
|
||||
" def add_batch_fn(data):\n",
|
||||
" return replay_buffer.add_batch(training_data_spec_transformation_fn(data))\n",
|
||||
"\n",
|
||||
" else:\n",
|
||||
" add_batch_fn = replay_buffer.add_batch\n",
|
||||
"\n",
|
||||
@@ -742,10 +753,12 @@
|
||||
" env=environment,\n",
|
||||
" policy=agent.collect_policy,\n",
|
||||
" num_steps=steps_per_loop * environment.batch_size,\n",
|
||||
" observers=observers)\n",
|
||||
" observers=observers,\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
" training_loop = trainer.get_training_loop_fn(\n",
|
||||
" driver, replay_buffer, agent, steps_per_loop)\n",
|
||||
" driver, replay_buffer, agent, steps_per_loop\n",
|
||||
" )\n",
|
||||
" saver = policy_saver.PolicySaver(agent.policy)\n",
|
||||
"\n",
|
||||
" for _ in range(training_loops):\n",
|
||||
@@ -783,7 +796,8 @@
|
||||
" environment=environment,\n",
|
||||
" training_loops=TRAINING_LOOPS,\n",
|
||||
" steps_per_loop=STEPS_PER_LOOP,\n",
|
||||
" additional_metrics=metrics)\n",
|
||||
" additional_metrics=metrics,\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"tf.profiler.experimental.stop()"
|
||||
]
|
||||
@@ -1092,11 +1106,15 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"RUN_HYPERPARAMETER_TUNING = True # Execute hyperparameter tuning instead of regular training.\n",
|
||||
"RUN_HYPERPARAMETER_TUNING = (\n",
|
||||
" True # Execute hyperparameter tuning instead of regular training.\n",
|
||||
")\n",
|
||||
"TRAIN_WITH_BEST_HYPERPARAMETERS = False # Do not train.\n",
|
||||
"\n",
|
||||
"HPTUNING_RESULT_DIR = \"hptuning/\" # @param {type: \"string\"} Directory to store the best hyperparameter(s) in `BUCKET_NAME` and locally (temporarily).\n",
|
||||
"HPTUNING_RESULT_PATH = os.path.join(HPTUNING_RESULT_DIR, \"result.json\") # @param {type: \"string\"} Path to the file containing the best hyperparameter(s)."
|
||||
"HPTUNING_RESULT_PATH = os.path.join(\n",
|
||||
" HPTUNING_RESULT_DIR, \"result.json\"\n",
|
||||
") # @param {type: \"string\"} Path to the file containing the best hyperparameter(s)."
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -1124,7 +1142,7 @@
|
||||
" image_uri: str,\n",
|
||||
" args: List[str],\n",
|
||||
" location: str = \"us-central1\",\n",
|
||||
" api_endpoint: str = \"us-central1-aiplatform.googleapis.com\"\n",
|
||||
" api_endpoint: str = \"us-central1-aiplatform.googleapis.com\",\n",
|
||||
") -> None:\n",
|
||||
" \"\"\"Creates a hyperparameter tuning job using a custom container.\n",
|
||||
"\n",
|
||||
@@ -1197,8 +1215,8 @@
|
||||
"\n",
|
||||
" # Create job\n",
|
||||
" response = client.create_hyperparameter_tuning_job(\n",
|
||||
" parent=parent,\n",
|
||||
" hyperparameter_tuning_job=hyperparameter_tuning_job)\n",
|
||||
" parent=parent, hyperparameter_tuning_job=hyperparameter_tuning_job\n",
|
||||
" )\n",
|
||||
" job_id = response.name.split(\"/\")[-1]\n",
|
||||
" print(\"Job ID:\", job_id)\n",
|
||||
" print(\"Job config:\", response)\n",
|
||||
@@ -1242,7 +1260,8 @@
|
||||
" image_uri=f\"gcr.io/{PROJECT_ID}/{HPTUNING_TRAINING_CONTAINER}:latest\",\n",
|
||||
" args=args,\n",
|
||||
" location=REGION,\n",
|
||||
" api_endpoint=f\"{REGION}-aiplatform.googleapis.com\")"
|
||||
" api_endpoint=f\"{REGION}-aiplatform.googleapis.com\",\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -1292,7 +1311,8 @@
|
||||
" name = client.hyperparameter_tuning_job_path(\n",
|
||||
" project=project,\n",
|
||||
" location=location,\n",
|
||||
" hyperparameter_tuning_job=hyperparameter_tuning_job_id)\n",
|
||||
" hyperparameter_tuning_job=hyperparameter_tuning_job_id,\n",
|
||||
" )\n",
|
||||
" response = client.get_hyperparameter_tuning_job(name=name)\n",
|
||||
" return response"
|
||||
]
|
||||
@@ -1313,7 +1333,8 @@
|
||||
" location=REGION,\n",
|
||||
" api_endpoint=f\"{REGION}-aiplatform.googleapis.com\")\n",
|
||||
" if response.state.name == 'JOB_STATE_SUCCEEDED':\n",
|
||||
" print(\"Job succeeded.\\nJob Time:\", response.update_time - response.create_time)\n",
|
||||
" print(\"Job succeeded.\n",
|
||||
"Job Time:\", response.update_time - response.create_time)\n",
|
||||
" trials = response.trials\n",
|
||||
" print(\"Trials:\", trials)\n",
|
||||
" break\n",
|
||||
@@ -1348,8 +1369,8 @@
|
||||
"if trials:\n",
|
||||
" # Dict mapping from metric names to the best metric values seen so far\n",
|
||||
" best_objective_values = dict.fromkeys(\n",
|
||||
" [metric.metric_id for metric in trials[0].final_measurement.metrics],\n",
|
||||
" -np.inf)\n",
|
||||
" [metric.metric_id for metric in trials[0].final_measurement.metrics], -np.inf\n",
|
||||
" )\n",
|
||||
" # Dict mapping from metric names to a list of the best combination(s) of\n",
|
||||
" # hyperparameter(s). Each combination is a dict mapping from hyperparameter\n",
|
||||
" # names to their values.\n",
|
||||
@@ -1358,12 +1379,13 @@
|
||||
" # `final_measurement` and `parameters` are `RepeatedComposite` objects.\n",
|
||||
" # Reference the structure above to extract the value of your interest.\n",
|
||||
" for metric in trial.final_measurement.metrics:\n",
|
||||
" params = {\n",
|
||||
" param.parameter_id: param.value for param in trial.parameters}\n",
|
||||
" params = {param.parameter_id: param.value for param in trial.parameters}\n",
|
||||
" if metric.value > best_objective_values[metric.metric_id]:\n",
|
||||
" best_params[metric.metric_id] = [params]\n",
|
||||
" elif metric.value == best_objective_values[metric.metric_id]:\n",
|
||||
" best_params[param.parameter_id].append(params) # Handle cases where multiple hyperparameter values lead to the same performance.\n",
|
||||
" best_params[param.parameter_id].append(\n",
|
||||
" params\n",
|
||||
" ) # Handle cases where multiple hyperparameter values lead to the same performance.\n",
|
||||
" print(\"Best hyperparameter value(s):\")\n",
|
||||
" for metric, params in best_params.items():\n",
|
||||
" print(f\"Metric={metric}: {sorted(params)}\")\n",
|
||||
@@ -1443,7 +1465,9 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"PREDICTION_CONTAINER = \"prediction-custom-container\" # @param {type:\"string\"} Name of the container image."
|
||||
"PREDICTION_CONTAINER = (\n",
|
||||
" \"prediction-custom-container\" # @param {type:\"string\"} Name of the container image.\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -1475,7 +1499,7 @@
|
||||
" machineType: 'E2_HIGHCPU_8'\"\"\".format(\n",
|
||||
" PROJECT_ID=PROJECT_ID,\n",
|
||||
" PREDICTION_CONTAINER=PREDICTION_CONTAINER,\n",
|
||||
" ARTIFACTS_DIR=ARTIFACTS_DIR\n",
|
||||
" ARTIFACTS_DIR=ARTIFACTS_DIR,\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"with open(\"cloudbuild.yaml\", \"w\") as fp:\n",
|
||||
@@ -1592,8 +1616,12 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"RUN_HYPERPARAMETER_TUNING = False # Execute regular training instead of hyperparameter tuning.\n",
|
||||
"TRAIN_WITH_BEST_HYPERPARAMETERS = True # @param {type:\"bool\"} Whether to use learned hyperparameters in training."
|
||||
"RUN_HYPERPARAMETER_TUNING = (\n",
|
||||
" False # Execute regular training instead of hyperparameter tuning.\n",
|
||||
")\n",
|
||||
"TRAIN_WITH_BEST_HYPERPARAMETERS = (\n",
|
||||
" True # @param {type:\"bool\"} Whether to use learned hyperparameters in training.\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -1633,10 +1661,12 @@
|
||||
"job = aiplatform.CustomContainerTrainingJob(\n",
|
||||
" display_name=\"train-movielens\",\n",
|
||||
" container_uri=f\"gcr.io/{PROJECT_ID}/{HPTUNING_TRAINING_CONTAINER}:latest\",\n",
|
||||
" command=[\"python3\", \"-m\", \"src.training.task\"] + args, # Pass in training arguments, including hyperparameters.\n",
|
||||
" command=[\"python3\", \"-m\", \"src.training.task\"]\n",
|
||||
" + args, # Pass in training arguments, including hyperparameters.\n",
|
||||
" model_serving_container_image_uri=f\"gcr.io/{PROJECT_ID}/{PREDICTION_CONTAINER}:latest\",\n",
|
||||
" model_serving_container_predict_route=\"/predict\",\n",
|
||||
" model_serving_container_health_route=\"/health\")\n",
|
||||
" model_serving_container_health_route=\"/health\",\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"print(\"Training Spec:\", job._managed_model)\n",
|
||||
"\n",
|
||||
@@ -1645,7 +1675,8 @@
|
||||
" replica_count=1,\n",
|
||||
" machine_type=\"n1-standard-4\",\n",
|
||||
" accelerator_type=\"ACCELERATOR_TYPE_UNSPECIFIED\",\n",
|
||||
" accelerator_count=0)"
|
||||
" accelerator_count=0,\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -1784,7 +1815,7 @@
|
||||
"! gcloud ai models delete $model.name --quiet\n",
|
||||
"\n",
|
||||
"# Delete Cloud Storage objects that were created\n",
|
||||
"! gsutil -m rm -r $ARTIFACTS_DIR"
|
||||
"! gcloud storage rm --recursive $ARTIFACTS_DIR"
|
||||
]
|
||||
}
|
||||
],
|
||||
|
||||
+2
-2
@@ -324,7 +324,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! gsutil ls $gcs_output_uri_prefix"
|
||||
"! gcloud storage ls $gcs_output_uri_prefix"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -344,7 +344,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! gsutil rm -rf $gcs_output_uri_prefix"
|
||||
"! gcloud storage rm --recursive --continue-on-error $gcs_output_uri_prefix"
|
||||
]
|
||||
}
|
||||
],
|
||||
|
||||
+2
-2
@@ -328,7 +328,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! gsutil ls $gcs_output_uri_prefix"
|
||||
"! gcloud storage ls $gcs_output_uri_prefix"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -348,7 +348,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! gsutil rm -rf $gcs_output_uri_prefix"
|
||||
"! gcloud storage rm --recursive --continue-on-error $gcs_output_uri_prefix"
|
||||
]
|
||||
}
|
||||
],
|
||||
|
||||
+2
-2
@@ -341,7 +341,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! gsutil ls $gcs_output_uri_prefix"
|
||||
"! gcloud storage ls $gcs_output_uri_prefix"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -361,7 +361,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! gsutil rm -rf $gcs_output_uri_prefix"
|
||||
"! gcloud storage rm --recursive --continue-on-error $gcs_output_uri_prefix"
|
||||
]
|
||||
}
|
||||
],
|
||||
|
||||
+2
-2
@@ -2,9 +2,9 @@ dllogger@git+https://github.com/NVIDIA/dllogger@v1.0.0
|
||||
|
||||
# Fixing these libraries versions to avoid conflicting or broken packages.
|
||||
immutabledict==4.2.1
|
||||
protobuf==4.25.8
|
||||
protobuf==5.29.6
|
||||
opencv-python-headless==4.11.0.86
|
||||
docutils==0.16
|
||||
urllib3==2.5.0
|
||||
urllib3==2.6.3
|
||||
google-cloud-storage==3.0.0
|
||||
retrying
|
||||
|
||||
+3
-3
@@ -31,8 +31,8 @@ packaging==25.0
|
||||
pathspec==0.12.1
|
||||
platformdirs==4.3.8
|
||||
proto-plus==1.26.1
|
||||
protobuf==5.29.4
|
||||
pyasn1==0.6.1
|
||||
protobuf==5.29.6
|
||||
pyasn1==0.6.2
|
||||
pyasn1_modules==0.4.2
|
||||
pydantic==2.11.4
|
||||
pydantic_core==2.33.2
|
||||
@@ -45,5 +45,5 @@ six==1.17.0
|
||||
sniffio==1.3.1
|
||||
typing-inspection==0.4.0
|
||||
typing_extensions==4.13.2
|
||||
urllib3==2.4.0
|
||||
urllib3==2.6.3
|
||||
websockets==15.0.1
|
||||
+7
-7
@@ -66,7 +66,7 @@ mkdir -p "$local_folder"
|
||||
mkdir -p "$output_folder"
|
||||
|
||||
# Download the content from the GCS URI
|
||||
gsutil -m cp -r "$gcs_dataset_path"/* "$local_folder/"
|
||||
gcloud storage cp --recursive "$gcs_dataset_path"/* "$local_folder/"
|
||||
|
||||
# Process files in the local folder
|
||||
for file in "$local_folder"/*; do
|
||||
@@ -122,23 +122,23 @@ cp -r "$output_folder" "$images_folder"/images_2
|
||||
pushd "$images_folder"/images_2
|
||||
ls | xargs -P 8 -I {} mogrify -resize 50% {}
|
||||
popd
|
||||
gsutil -m cp -r "$images_folder"/images_2/* "$gcs_experiment_path"/data/images_2
|
||||
gcloud storage cp --recursive "$images_folder"/images_2/* "$gcs_experiment_path"/data/images_2
|
||||
|
||||
cp -r "$output_folder" "$images_folder"/images_4
|
||||
pushd "$images_folder"/images_4
|
||||
ls | xargs -P 8 -I {} mogrify -resize 25% {}
|
||||
popd
|
||||
gsutil -m cp -r "$images_folder"/images_4/* "$gcs_experiment_path"/data/images_4
|
||||
gcloud storage cp --recursive "$images_folder"/images_4/* "$gcs_experiment_path"/data/images_4
|
||||
|
||||
cp -r "$output_folder" "$images_folder"/images_8
|
||||
pushd "$images_folder"/images_8
|
||||
ls | xargs -P 8 -I {} mogrify -resize 12.5% {}
|
||||
popd
|
||||
gsutil -m cp "$images_folder"/images_8/* "$gcs_experiment_path"/data/images_8
|
||||
gcloud storage cp "$images_folder"/images_8/* "$gcs_experiment_path"/data/images_8
|
||||
|
||||
# Copy images and sparse reconstruction files to gcs experiment folder.
|
||||
gsutil -m cp "$images_folder"/images/* "$gcs_experiment_path"/data/images
|
||||
gsutil -m cp -r "$local_folder"/sparse "$gcs_experiment_path"/data
|
||||
gsutil -m cp "$local_folder"/database.db "$gcs_experiment_path"/data
|
||||
gcloud storage cp "$images_folder"/images/* "$gcs_experiment_path"/data/images
|
||||
gcloud storage cp --recursive "$local_folder"/sparse "$gcs_experiment_path"/data
|
||||
gcloud storage cp "$local_folder"/database.db "$gcs_experiment_path"/data
|
||||
|
||||
echo "Processing complete."
|
||||
@@ -99,14 +99,14 @@ create_dir_if_not_exists "$CHECKPOINTS_PATH"
|
||||
touch "$local_experiment_path/$exp_folder_name/log_render.txt"
|
||||
|
||||
# Copy experiment from GCS bucket to local
|
||||
gsutil -m cp -r "${args[-gcs_experiment_path]}/data" "$local_experiment_path/$exp_folder_name" || exit 1
|
||||
gsutil -m cp -r "${args[-gcs_experiment_path]}/checkpoints/${training_job_name}/*" "$CHECKPOINTS_PATH" || exit 1
|
||||
gcloud storage cp --recursive "${args[-gcs_experiment_path]}/data" "$local_experiment_path/$exp_folder_name" || exit 1
|
||||
gcloud storage cp --recursive "${args[-gcs_experiment_path]}/checkpoints/${training_job_name}/*" "$CHECKPOINTS_PATH" || exit 1
|
||||
|
||||
# Check and copy keyframes file.
|
||||
if [[ -n ${args[-gcs_keyframes_file]} ]]; then
|
||||
keyframes_file_basename=$(basename "${args[-gcs_keyframes_file]}")
|
||||
local_keyframes_file="$local_dataset_path/$keyframes_file_basename"
|
||||
gsutil cp "${args[-gcs_keyframes_file]}" "$local_keyframes_file" || exit 1
|
||||
gcloud storage cp "${args[-gcs_keyframes_file]}" "$local_keyframes_file" || exit 1
|
||||
echo "Local keyframe file: $local_keyframes_file"
|
||||
launch_rendering "$local_keyframes_file"
|
||||
else
|
||||
@@ -114,4 +114,4 @@ else
|
||||
fi
|
||||
|
||||
# Copy rendered data back to GCS.
|
||||
gsutil -m cp -r "$OUTPUT_RENDER_PATH" "${args[-gcs_experiment_path]}/render/${rendering_job_name}"
|
||||
gcloud storage cp --recursive "$OUTPUT_RENDER_PATH" "${args[-gcs_experiment_path]}/render/${rendering_job_name}"
|
||||
@@ -74,7 +74,7 @@ create_dir_if_not_exists "$local_experiment_path"
|
||||
create_dir_if_not_exists "$local_experiment_path/$scene_folder_name"
|
||||
|
||||
# Copy experiment from GCS bucket to local.
|
||||
gsutil -m cp -r "${gcs_experiment_path}/data" "$local_experiment_path/$scene_folder_name" || exit 1
|
||||
gcloud storage cp --recursive "${gcs_experiment_path}/data" "$local_experiment_path/$scene_folder_name" || exit 1
|
||||
|
||||
echo "GCS Experiment: $gcs_experiment_path"
|
||||
echo "Gin Config File: $gin_config_file"
|
||||
@@ -89,6 +89,6 @@ accelerate launch train.py --gin_configs="$gin_config_file" \
|
||||
--gin_bindings="Config.factor = ${factor}" \
|
||||
--gin_bindings="Config.max_steps = ${max_training_steps}"
|
||||
|
||||
gsutil -m rm -r "${gcs_experiment_path}/checkpoints/${training_job_name}"
|
||||
gsutil -m cp -r "$local_experiment_path/$scene_folder_name/config.gin" "${gcs_experiment_path}/${training_job_name}_config.gin"
|
||||
gsutil -m cp -r "$local_experiment_path/$scene_folder_name/checkpoints/*/*" "${gcs_experiment_path}/checkpoints/${training_job_name}"
|
||||
gcloud storage rm --recursive "${gcs_experiment_path}/checkpoints/${training_job_name}"
|
||||
gcloud storage cp --recursive "$local_experiment_path/$scene_folder_name/config.gin" "${gcs_experiment_path}/${training_job_name}_config.gin"
|
||||
gcloud storage cp --recursive "$local_experiment_path/$scene_folder_name/checkpoints/*/*" "${gcs_experiment_path}/checkpoints/${training_job_name}"
|
||||
+3
-3
@@ -102,10 +102,10 @@ def download_gcs_uri_to_local(
|
||||
if not os.path.exists(destination_dir):
|
||||
os.mkdir(destination_dir)
|
||||
subprocess.check_output([
|
||||
"gsutil",
|
||||
"-m",
|
||||
"gcloud",
|
||||
"storage",
|
||||
"cp",
|
||||
"-r",
|
||||
"--recursive",
|
||||
gcs_uri,
|
||||
destination_dir,
|
||||
])
|
||||
|
||||
@@ -26,6 +26,9 @@
|
||||
/vertex_endpoints/find_ideal_machine_type/find_ideal_machine_type/find_ideal_machine_type.ipynb @entrpn
|
||||
/vertex_endpoints/nvidia-triton/nvidia-triton-custom-container-prediction.ipynb @RajeshThallam
|
||||
/vertex_endpoints/optimized_tensorflow_runtime @vlasenkoalexey
|
||||
/notebooks/community/alphagenome/cloudai_alphagenome_vai_quickstart.ipynb @dpanigra
|
||||
/notebooks/community/alphagenome/cloudai_alphagenome_finetune.ipynb @dpanigra
|
||||
/notebooks/community/weathernext/weathernext_2_early_access_program.ipynb @dpanigra
|
||||
/notebooks/community/ml_ops/stage2/get_started_with_visionapi_and_automl.ipynb @mansari
|
||||
/notebooks/community/neo4j/graph_paysim.ipynb @benofben @laeg
|
||||
/notebooks/community/ml_ops/stage1/get_started_with_visionapi_and_vertex_datasets.ipynb @mansari
|
||||
|
||||
@@ -0,0 +1,552 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "f705f4be70e9"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Copyright 2025 Google LLC\n",
|
||||
"#\n",
|
||||
"# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
|
||||
"# you may not use this file except in compliance with the License.\n",
|
||||
"# You may obtain a copy of the License at\n",
|
||||
"#\n",
|
||||
"# https://www.apache.org/licenses/LICENSE-2.0\n",
|
||||
"#\n",
|
||||
"# Unless required by applicable law or agreed to in writing, software\n",
|
||||
"# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
|
||||
"# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
|
||||
"# See the License for the specific language governing permissions and\n",
|
||||
"# limitations under the License."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "9a05a17e4e2f"
|
||||
},
|
||||
"source": [
|
||||
"# Deploy ADK Agents to Vertex AI Agent Engine Using Inline Source\n",
|
||||
"\n",
|
||||
"<table><tbody><tr>\n",
|
||||
" <td style=\"text-align: center\">\n",
|
||||
" <a href=\"https://console.cloud.google.com/vertex-ai/colab/import/https:%2F%2Fraw.githubusercontent.com%2FGoogleCloudPlatform%2Fvertex-ai-samples%2Fmain%2Fnotebooks%2Fcommunity%2Fagent_engine%2Fadk_inline_source_deployment.ipynb\">\n",
|
||||
" <img alt=\"Google Cloud Colab Enterprise logo\" src=\"https://lh3.googleusercontent.com/JmcxdQi-qOpctIvWKgPtrzZdJJK-J3sWE1RsfjZNwshCFgE_9fULcNpuXYTilIR2hjwN\" width=\"32px\"><br> Run in Colab Enterprise\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
" <td style=\"text-align: center\">\n",
|
||||
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/agent_engine/adk_inline_source_deployment.ipynb\">\n",
|
||||
" <img alt=\"GitHub logo\" src=\"https://github.githubassets.com/assets/GitHub-Mark-ea2971cee799.png\" width=\"32px\"><br> View on GitHub\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
"</tr></tbody></table>"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "e6e9014cd53a"
|
||||
},
|
||||
"source": [
|
||||
"## Overview\n",
|
||||
"\n",
|
||||
"This notebook demonstrates how to deploy [Google Agent Development Kit (ADK)](https://google.github.io/adk-docs/) agents to [Vertex AI Agent Engine](https://cloud.google.com/vertex-ai/generative-ai/docs/agent-engine/overview) using **inline source deployment**.\n",
|
||||
"\n",
|
||||
"I built this pattern while developing a multi-agent system with ADK. The approach is also featured in Google's [Agent Starter Pack community showcase](https://github.com/GoogleCloudPlatform/agent-starter-pack#community-showcase).\n",
|
||||
"\n",
|
||||
"Inline source deployment sends your agent's Python source code directly to Agent Engine, avoiding serialization issues that can occur with the default pickle-based deployment.\n",
|
||||
"\n",
|
||||
"### Why Inline Source Deployment?\n",
|
||||
"\n",
|
||||
"| Approach | How it works | Best for |\n",
|
||||
"|----------|--------------|----------|\n",
|
||||
"| Default (pickle) | Serializes agent object | Simple agents, quick prototypes |\n",
|
||||
"| **Inline source** | Sends Python source directly | Complex agents, custom dependencies |\n",
|
||||
"\n",
|
||||
"Inline source deployment is useful when:\n",
|
||||
"- Your agent uses custom classes that don't serialize well\n",
|
||||
"- You need precise control over the deployment environment\n",
|
||||
"- You want to avoid pickle compatibility issues across Python versions\n",
|
||||
"\n",
|
||||
"### Objectives\n",
|
||||
"\n",
|
||||
"- Create an ADK agent with tools\n",
|
||||
"- Deploy to Agent Engine using inline source\n",
|
||||
"- Test the deployed agent\n",
|
||||
"- Clean up resources\n",
|
||||
"\n",
|
||||
"### Costs\n",
|
||||
"\n",
|
||||
"This tutorial uses billable components of Google Cloud:\n",
|
||||
"\n",
|
||||
"* Vertex AI Agent Engine\n",
|
||||
"* Vertex AI (Gemini API)\n",
|
||||
"\n",
|
||||
"Learn about [Vertex AI pricing](https://cloud.google.com/vertex-ai/pricing) and use the [Pricing Calculator](https://cloud.google.com/products/calculator/) to generate a cost estimate based on your projected usage."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "4d998a5140b2"
|
||||
},
|
||||
"source": [
|
||||
"## Get started"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "c832028166e3"
|
||||
},
|
||||
"source": [
|
||||
"### Install required packages"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "6b2e64d59bf4"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Define requirements once for reuse\n",
|
||||
"REQUIREMENTS = [\n",
|
||||
" \"google-cloud-aiplatform[adk,agent_engines]>=1.88.0\",\n",
|
||||
" \"google-adk>=1.18.0\",\n",
|
||||
"]\n",
|
||||
"\n",
|
||||
"!pip install --upgrade --quiet {\" \".join(REQUIREMENTS)}"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "2f2e1483e7a8"
|
||||
},
|
||||
"source": [
|
||||
"### Restart runtime\n",
|
||||
"\n",
|
||||
"After installing packages, restart the runtime to ensure the new packages are loaded."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "d76628df8180"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import IPython\n",
|
||||
"\n",
|
||||
"app = IPython.Application.instance()\n",
|
||||
"app.kernel.do_shutdown(True)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "586675a64f57"
|
||||
},
|
||||
"source": [
|
||||
"### Authenticate your notebook environment (Colab only)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "2082bbcb771c"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import sys\n",
|
||||
"\n",
|
||||
"if \"google.colab\" in sys.modules:\n",
|
||||
" from google.colab import auth\n",
|
||||
"\n",
|
||||
" auth.authenticate_user()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "296a2345bc1a"
|
||||
},
|
||||
"source": [
|
||||
"### Set Google Cloud project information"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "abafb08d10b1"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import os\n",
|
||||
"\n",
|
||||
"PROJECT_ID = \"[your-project-id]\" # @param {type: \"string\", placeholder: \"[your-project-id]\", isTemplate: true}\n",
|
||||
"if not PROJECT_ID or PROJECT_ID == \"[your-project-id]\":\n",
|
||||
" PROJECT_ID = str(os.environ.get(\"GOOGLE_CLOUD_PROJECT\"))\n",
|
||||
"\n",
|
||||
"LOCATION = \"us-central1\" # @param {type:\"string\"}\n",
|
||||
"\n",
|
||||
"STAGING_BUCKET = f\"gs://{PROJECT_ID}-agent-engine-staging\" # @param {type:\"string\"}"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "c7863dcf6a38"
|
||||
},
|
||||
"source": [
|
||||
"### Create staging bucket\n",
|
||||
"\n",
|
||||
"Agent Engine requires a staging bucket to store deployment artifacts."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "17ece3f9fb6e"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"!gsutil mb -l {LOCATION} -p {PROJECT_ID} {STAGING_BUCKET} 2>/dev/null || echo \"Bucket already exists\""
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "1029f595561d"
|
||||
},
|
||||
"source": [
|
||||
"### Initialize Vertex AI"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "cac3c1b09203"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import vertexai\n",
|
||||
"\n",
|
||||
"vertexai.init(\n",
|
||||
" project=PROJECT_ID,\n",
|
||||
" location=LOCATION,\n",
|
||||
" staging_bucket=STAGING_BUCKET,\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "46fb887b5ea3"
|
||||
},
|
||||
"source": [
|
||||
"## Create an ADK Agent\n",
|
||||
"\n",
|
||||
"First, let's create a simple ADK agent with a tool. This agent can answer questions and get the current time."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "9d0dd0808c69"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import datetime\n",
|
||||
"from zoneinfo import ZoneInfo, ZoneInfoNotFoundError\n",
|
||||
"\n",
|
||||
"from google.adk.agents import LlmAgent\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def get_current_time(timezone: str = \"UTC\") -> str:\n",
|
||||
" \"\"\"Get the current time in the specified timezone.\n",
|
||||
"\n",
|
||||
" Args:\n",
|
||||
" timezone: IANA timezone name (e.g., 'America/New_York', 'UTC')\n",
|
||||
"\n",
|
||||
" Returns:\n",
|
||||
" Current time as a formatted string\n",
|
||||
" \"\"\"\n",
|
||||
" try:\n",
|
||||
" tz = ZoneInfo(timezone)\n",
|
||||
" now = datetime.datetime.now(tz)\n",
|
||||
" return (\n",
|
||||
" f\"The current time in {timezone} is {now.strftime('%Y-%m-%d %H:%M:%S %Z')}\"\n",
|
||||
" )\n",
|
||||
" except ZoneInfoNotFoundError:\n",
|
||||
" return f\"Error: Timezone '{timezone}' not found. Please use a valid IANA timezone name (e.g., 'America/New_York', 'UTC').\"\n",
|
||||
" except Exception as e:\n",
|
||||
" return f\"An unexpected error occurred while getting time for timezone {timezone}: {e}\"\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"# Create the agent\n",
|
||||
"agent = LlmAgent(\n",
|
||||
" name=\"time_agent\",\n",
|
||||
" model=\"gemini-1.5-flash-001\",\n",
|
||||
" instruction=\"You are a helpful assistant. Use the get_current_time tool when asked about the time.\",\n",
|
||||
" tools=[get_current_time],\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"print(f\"Created agent: {agent.name}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "ad0b9cc230ca"
|
||||
},
|
||||
"source": [
|
||||
"### Test the agent locally\n",
|
||||
"\n",
|
||||
"Before deploying, let's test the agent locally to make sure it works."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "1d3d71bde31e"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from google.adk import Runner\n",
|
||||
"from google.adk.sessions import InMemorySessionService\n",
|
||||
"from google.genai import types\n",
|
||||
"\n",
|
||||
"# Create a local runner\n",
|
||||
"runner = Runner(\n",
|
||||
" agent=agent,\n",
|
||||
" app_name=\"test_app\",\n",
|
||||
" session_service=InMemorySessionService(),\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"# Create a session and test\n",
|
||||
"session = runner.session_service.create_session(\n",
|
||||
" app_name=\"test_app\",\n",
|
||||
" user_id=\"test_user\",\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"# Send a test message\n",
|
||||
"response = runner.run(\n",
|
||||
" user_id=\"test_user\",\n",
|
||||
" session_id=session.id,\n",
|
||||
" new_message=types.Content(\n",
|
||||
" role=\"user\",\n",
|
||||
" parts=[types.Part(text=\"What time is it in New York?\")],\n",
|
||||
" ),\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"# Print the response\n",
|
||||
"for event in response:\n",
|
||||
" if hasattr(event, \"content\") and event.content:\n",
|
||||
" for part in event.content.parts:\n",
|
||||
" if hasattr(part, \"text\") and part.text:\n",
|
||||
" print(part.text)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "06bdc3c10ea6"
|
||||
},
|
||||
"source": [
|
||||
"## Deploy Using Inline Source\n",
|
||||
"\n",
|
||||
"Now let's deploy the agent to Agent Engine using inline source deployment.\n",
|
||||
"\n",
|
||||
"The key difference from default deployment is using `AgentEngineApp.from_agent()` which packages the source code directly."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "d641a5e29291"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from vertexai import agent_engines\n",
|
||||
"from vertexai.agent_engines import AgentEngineApp\n",
|
||||
"\n",
|
||||
"# Create the app from the agent (inline source)\n",
|
||||
"app = AgentEngineApp.from_agent(\n",
|
||||
" agent=agent,\n",
|
||||
" enable_tracing=True,\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"print(\"App created successfully\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "cfd8fddbf181"
|
||||
},
|
||||
"source": [
|
||||
"### Deploy to Agent Engine"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "ebd6642375a8"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Deploy the app\n",
|
||||
"# This may take a few minutes\n",
|
||||
"remote_app = agent_engines.create(\n",
|
||||
" app,\n",
|
||||
" requirements=REQUIREMENTS,\n",
|
||||
" display_name=\"time-agent-inline\",\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"print(f\"Deployed agent: {remote_app.resource_name}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "43b739d2f980"
|
||||
},
|
||||
"source": [
|
||||
"## Test the Deployed Agent\n",
|
||||
"\n",
|
||||
"Now let's test the deployed agent by sending it a query."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "8ec0598a1138"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Create a session with the remote agent\n",
|
||||
"remote_session = remote_app.create_session(user_id=\"test_user\")\n",
|
||||
"\n",
|
||||
"print(f\"Created session: {remote_session['id']}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "f85603f0e4fd"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Query the agent\n",
|
||||
"response = remote_app.stream_query(\n",
|
||||
" user_id=\"test_user\",\n",
|
||||
" session_id=remote_session[\"id\"],\n",
|
||||
" message=\"What time is it in Tokyo?\",\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"# Print the response\n",
|
||||
"for chunk in response:\n",
|
||||
" if hasattr(chunk, \"content\") and chunk.content:\n",
|
||||
" for part in chunk.content.parts:\n",
|
||||
" if hasattr(part, \"text\") and part.text:\n",
|
||||
" print(part.text, end=\"\")\n",
|
||||
"\n",
|
||||
"print() # Newline at end"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "e70d97acf3a3"
|
||||
},
|
||||
"source": [
|
||||
"## Cleaning up\n",
|
||||
"\n",
|
||||
"Delete the deployed agent to avoid ongoing charges."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "6d2d34a6ba7c"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Delete the deployed agent\n",
|
||||
"delete_agent = False # @param {type:\"boolean\"}\n",
|
||||
"\n",
|
||||
"if delete_agent:\n",
|
||||
" remote_app.delete()\n",
|
||||
" print(\"Agent deleted successfully\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "35870ecf063a"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Delete the staging bucket\n",
|
||||
"delete_bucket = False # @param {type:\"boolean\"}\n",
|
||||
"\n",
|
||||
"if delete_bucket:\n",
|
||||
" !gsutil rm -r {STAGING_BUCKET}\n",
|
||||
" print(\"Bucket deleted successfully\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "11f2cc08759a"
|
||||
},
|
||||
"source": [
|
||||
"## Summary\n",
|
||||
"\n",
|
||||
"In this notebook, you learned how to:\n",
|
||||
"\n",
|
||||
"1. Create an ADK agent with tools\n",
|
||||
"2. Test the agent locally\n",
|
||||
"3. Deploy to Agent Engine using inline source with `AgentEngineApp.from_agent()`\n",
|
||||
"4. Query the deployed agent\n",
|
||||
"\n",
|
||||
"### Next steps\n",
|
||||
"\n",
|
||||
"- Learn more about [ADK](https://google.github.io/adk-docs/)\n",
|
||||
"- Explore [Agent Engine documentation](https://cloud.google.com/vertex-ai/generative-ai/docs/agent-engine/overview)\n",
|
||||
"- Try adding more complex tools and multi-agent orchestration"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"colab": {
|
||||
"name": "adk_inline_source_deployment.ipynb",
|
||||
"toc_visible": true
|
||||
},
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3",
|
||||
"name": "python3"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 0
|
||||
}
|
||||
@@ -0,0 +1,97 @@
|
||||

|
||||
|
||||
# AlphaGenome
|
||||
[**Overview**](#overview) | [**Use Cases**](#use-cases) | [**Documentation**](#documentation) | [**Pricing**](#pricing) | [**Quick start inference**](#quick-start-inference) |
|
||||
[**Quick start finetune**](#quick-start-finetune)
|
||||
|
||||
## Overview
|
||||
**Disclaimer:** *Experimental*.
|
||||
|
||||
*The AlphaGenome Private Preview is a "Pre-GA Offering" subject to the "Pre-GA
|
||||
Offerings Terms" in the General Service Terms section of the Google Cloud
|
||||
[Service Specific Terms](https://cloud.google.com/terms/service-terms). It is
|
||||
also a “Generative AI Preview Product” as defined in and subject to the
|
||||
[Additional Terms for Generative AI Preview Products](https://cloud.google.com/trustedtester/aitos?e=48754805&hl=en).
|
||||
Pre-GA products are available "as is" and might have limited support. For more
|
||||
information, see the [launch stage](https://cloud.google.com/products?e=48754805#product-launch-stages)
|
||||
descriptions.*
|
||||
|
||||
Access to the AlphaGenome model capabilities requires application and approval.
|
||||
Users must be added to an allowlist to use the service.
|
||||
If you are interested in applying to the program, **Request Access** above.
|
||||
|
||||
|
||||
|
||||
AlphaGenome is Google DeepMind’s unifying model for deciphering the regulatory
|
||||
code within DNA sequences.
|
||||
|
||||
AlphaGenome offers multimodal predictions, encompassing diverse functional
|
||||
outputs such as gene expression, splicing patterns, chromatin features, and
|
||||
contact maps (see diagram below). The model analyzes DNA sequences of up to 1
|
||||
million base pairs in length and can deliver predictions at single base-pair
|
||||
resolution for most outputs.
|
||||
|
||||
Training data was sourced from large public consortia including
|
||||
[ENCODE](http://encodeproject.org/), [GTEx](https://www.gtexportal.org/),
|
||||
[4D Nucleome](https://4dnucleome.org/) and
|
||||
[FANTOM5](https://fantom.gsc.riken.jp/5/), which experimentally measured these
|
||||
properties covering important modalities of gene regulation across hundreds of
|
||||
human and mouse cell types and tissues.
|
||||
|
||||

|
||||
|
||||
## Use Cases
|
||||
* **Sequence-to-function predictions:** Predict multiple functional tracks (such as gene expression, splicing) from DNA sequences across a wide variety of tissues and cell types.
|
||||
|
||||
* **Variant effect scoring:** Assess the impact of genetic variants by comparing predictions for the reference and alternative alleles and summarising the differences between them.
|
||||
|
||||
* **Identify functional regions:** Use in silico mutagenesis (ISM) to identify functionally important regions in the DNA sequence.
|
||||
|
||||
* **Human and mouse capability:** Generate predictions for both human and mouse genomes.
|
||||
|
||||
## Documentation
|
||||
This API provides access to AlphaGenome, Google DeepMind's unifying model for
|
||||
deciphering the regulatory code within DNA sequences. AlphaGenome offers
|
||||
multimodal predictions, encompassing diverse functional outputs including gene
|
||||
expression, splicing patterns, chromatin features, and contact maps (see diagram
|
||||
below). The model analyzes up to 1 million base pairs of DNA sequence and can
|
||||
deliver predictions at single base-pair resolution for most modalities.
|
||||
AlphaGenome achieves state-of-the-art performance across a range of genomic
|
||||
prediction benchmarks, including diverse variant effect prediction tasks.
|
||||
|
||||
The Google Cloud API for AlphaGenome provides a way for Google Cloud customers
|
||||
to explore the AlphaGenome API for commercial use cases. This API is in private
|
||||
preview (Request Access above). Once allowlisted, customers can access the API
|
||||
directly or use the [colab](cloudai_alphagenome_vai_quickstart.ipynb).
|
||||
|
||||
### Acknowledgements
|
||||
|
||||
*Avsec, Ž., Latysheva, N., Cheng, J., Novati, G., Taylor, K. R., Ward, T., ... Kohli, P. (2025). AlphaGenome: advancing regulatory variant effect prediction with a unified DNA sequence model. bioRxiv.* [https://doi.org/10.1101/2025.06.25.661532](https://doi.org/10.1101/2025.06.25.661532)
|
||||
|
||||
### Contact
|
||||
If you have any questions on using these models on Google Cloud please contact:
|
||||
[alphagenome-cloud-external@google.com](mailto:alphagenome-cloud-external@google.com) or join the community [Discourse](https://www.alphagenomecommunity.com/) for more generic questions on AlphaGenome.
|
||||
|
||||
### Links
|
||||
|
||||
* Read our [paper](https://doi.org/10.1101/2025.06.25.661532)
|
||||
* Read our [blog post](https://deepmind.google/discover/blog/alphagenome-ai-for-better-understanding-the-genome)
|
||||
* Join the [community](https://www.alphagenomecommunity.com/)
|
||||
* Check out the [AlphaGenome 101 Video](https://youtu.be/Xbvloe13nak)
|
||||
|
||||
## Pricing
|
||||
Access to AlphaGenome on Vertex AI is currently restricted.
|
||||
To utilize these models via this service:
|
||||
|
||||
* You must **Request Access** using your Google contact.
|
||||
* Your application will be reviewed, and if approved, you will be **added to
|
||||
an allowlist**.
|
||||
* Only allowlisted users can access the API
|
||||
* **Pricing information** will be shared directly with users upon approval
|
||||
and placement on the allowlist.
|
||||
|
||||
## Quick start inference
|
||||
The quickest way to get started with the AlphaGenome in Google Cloud Platform is to run [our example notebook](cloudai_alphagenome_vai_quickstart.ipynb) in [Google Colab](https://colab.research.google.com/).
|
||||
|
||||
## Quick start finetune
|
||||
The quickest way to get started with the AlphaGenome fineutning in Google Cloud Platform is to run [our finetuning notebook](cloudai_alphagenome_finetune.ipynb) in [Google Cloud Platform Enterprise Colab](https://docs.cloud.google.com/colab/docs/introduction).
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because one or more lines are too long
+1
-1
@@ -42,7 +42,7 @@
|
||||
" </td>\n",
|
||||
" <td>\n",
|
||||
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/bigquery_ml/Anomaly_detection_in_Cloud_Audit_logs_with_BQML.ipynb\">\n",
|
||||
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
|
||||
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
|
||||
" View on GitHub\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
|
||||
@@ -52,7 +52,7 @@
|
||||
" </td>\n",
|
||||
" <td>\n",
|
||||
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/bigquery_ml/bq_ml_with_vision_translation_nlp.ipynb\">\n",
|
||||
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
|
||||
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
|
||||
" View on GitHub\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
|
||||
@@ -40,7 +40,7 @@
|
||||
" </td>\n",
|
||||
" <td>\n",
|
||||
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/bigquery_ml/bqml-online-prediction.ipynb\">\n",
|
||||
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
|
||||
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
|
||||
" View on GitHub\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
@@ -359,7 +359,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! gsutil mb -l {REGION} -p {PROJECT_ID} {BUCKET_URI}"
|
||||
"! gcloud storage buckets create --location={REGION} --project={PROJECT_ID} {BUCKET_URI}"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -1098,7 +1098,7 @@
|
||||
" ! bq rm -r -f $PROJECT_ID:$BQ_DATASET_NAME\n",
|
||||
"# delete the Cloud Storage bucket\n",
|
||||
"if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
|
||||
" ! gsutil -m rm -r $BUCKET_URI"
|
||||
" ! gcloud storage rm --recursive $BUCKET_URI"
|
||||
]
|
||||
}
|
||||
],
|
||||
|
||||
@@ -38,7 +38,7 @@
|
||||
" </td>\n",
|
||||
" <td>\n",
|
||||
" <a href=\"github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/cohere/cohere_embedding_with_matching_engine.ipynb\">\n",
|
||||
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
|
||||
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
|
||||
" View on GitHub\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
@@ -449,7 +449,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! gsutil mb -l $REGION $BUCKET_NAME"
|
||||
"! gcloud storage buckets create --location=$REGION $BUCKET_NAME"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -470,7 +470,7 @@
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# this will not return anything if the bucket is empty\n",
|
||||
"! gsutil ls -al $BUCKET_NAME"
|
||||
"! gcloud storage ls --all-versions --long $BUCKET_NAME"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -588,7 +588,7 @@
|
||||
"# NOTE: Everything in this GCS DIR will be DELETED before uploading the data.\n",
|
||||
"# A CommandException is expected if no data is present\n",
|
||||
"\n",
|
||||
"! gsutil rm -rf {BUCKET_NAME}/*"
|
||||
"! gcloud storage rm --recursive --continue-on-error {BUCKET_NAME}/*"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -599,7 +599,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! gsutil cp cohere_embeddings.json {BUCKET_NAME}/cohere_embeddings.json"
|
||||
"! gcloud storage cp cohere_embeddings.json {BUCKET_NAME}/cohere_embeddings.json"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -610,7 +610,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! gsutil ls {BUCKET_NAME}"
|
||||
"! gcloud storage ls {BUCKET_NAME}"
|
||||
]
|
||||
},
|
||||
{
|
||||
|
||||
@@ -41,7 +41,7 @@
|
||||
" </td>\n",
|
||||
" <td>\n",
|
||||
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/vertex_ai_experiments_classification.ipynb\">\n",
|
||||
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
|
||||
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
|
||||
" View on GitHub\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
@@ -537,7 +537,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! gsutil mb -l $REGION -p $PROJECT_ID $BUCKET_URI"
|
||||
"! gcloud storage buckets create --location=$REGION --project=$PROJECT_ID $BUCKET_URI"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -557,7 +557,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! gsutil ls -al $BUCKET_URI"
|
||||
"! gcloud storage ls --all-versions --long $BUCKET_URI"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -627,9 +627,9 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectCreator $BUCKET_URI\n",
|
||||
"! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n",
|
||||
"\n",
|
||||
"! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectViewer $BUCKET_URI"
|
||||
"! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -1736,7 +1736,7 @@
|
||||
"delete_bucket = False\n",
|
||||
"\n",
|
||||
"if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
|
||||
" ! gsutil rm -rf {BUCKET_URI}"
|
||||
" ! gcloud storage rm --recursive --continue-on-error {BUCKET_URI}"
|
||||
]
|
||||
}
|
||||
],
|
||||
|
||||
@@ -38,7 +38,7 @@
|
||||
" </td>\n",
|
||||
" <td>\n",
|
||||
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/explainable_ai/SDK_Custom_Container_XAI.ipynb\">\n",
|
||||
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
|
||||
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
|
||||
" View on GitHub\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
@@ -405,7 +405,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! gsutil mb -l $REGION -p $PROJECT_ID $BUCKET_URI"
|
||||
"! gcloud storage buckets create --location=$REGION --project=$PROJECT_ID $BUCKET_URI"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -425,7 +425,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! gsutil ls -al $BUCKET_URI"
|
||||
"! gcloud storage ls --all-versions --long $BUCKET_URI"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -698,7 +698,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"!gsutil cp app/model.joblib {BUCKET_URI}/{MODEL_ARTIFACT_DIR}/"
|
||||
"!gcloud storage cp app/model.joblib {BUCKET_URI}/{MODEL_ARTIFACT_DIR}/"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -1516,7 +1516,7 @@
|
||||
"\n",
|
||||
"delete_bucket = False\n",
|
||||
"if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
|
||||
" ! gsutil rm -rf {BUCKET_URI}"
|
||||
" ! gcloud storage rm --recursive --continue-on-error {BUCKET_URI}"
|
||||
]
|
||||
}
|
||||
],
|
||||
|
||||
@@ -637,20 +637,20 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Upload CSV data to Cloud Storage by passing gsutil commands to system\n",
|
||||
"# Upload CSV data to Cloud Storage by passing gcloud storage commands to system\n",
|
||||
"gcs_url <- paste0(\"gs://\", BUCKET_NAME, \"/\")\n",
|
||||
"\n",
|
||||
"command <- paste(\"gsutil mb\", gcs_url)\n",
|
||||
"command <- paste(\"gcloud storage buckets create\", gcs_url)\n",
|
||||
"\n",
|
||||
"system(command)\n",
|
||||
"\n",
|
||||
"gcs_data_dir <- paste0(\"gs://\", BUCKET_NAME, \"/data\")\n",
|
||||
"\n",
|
||||
"command <- paste(\"gsutil cp data/*_data.csv\", gcs_data_dir)\n",
|
||||
"command <- paste(\"gcloud storage cp data/*_data.csv\", gcs_data_dir)\n",
|
||||
"\n",
|
||||
"system(command)\n",
|
||||
"\n",
|
||||
"command <- paste(\"gsutil ls -l\", gcs_data_dir)\n",
|
||||
"command <- paste(\"gcloud storage ls --long\", gcs_data_dir)\n",
|
||||
"\n",
|
||||
"system(command, intern = TRUE)"
|
||||
]
|
||||
@@ -679,4 +679,4 @@
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 4
|
||||
}
|
||||
}
|
||||
@@ -38,7 +38,7 @@
|
||||
" </td>\n",
|
||||
" <td>\n",
|
||||
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/official/feature_store/gapic-feature-store.ipynb\">\n",
|
||||
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
|
||||
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
|
||||
" View on GitHub\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
|
||||
@@ -34,7 +34,7 @@
|
||||
"<table align=\"left\">\n",
|
||||
" <td>\n",
|
||||
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage6/get_started_vertex_feature_store_serving.ipynb\">\n",
|
||||
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
|
||||
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
|
||||
" View on GitHub\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
@@ -471,8 +471,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! gsutil mb -l $REGION $BUCKET_URI"
|
||||
]
|
||||
"! gcloud storage buckets create --location=$REGION $BUCKET_URI" ]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
@@ -491,8 +490,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! gsutil ls -al $BUCKET_URI"
|
||||
]
|
||||
"! gcloud storage ls --all-versions --long $BUCKET_URI" ]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
@@ -511,8 +509,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! gsutil uniformbucketlevelaccess set on {BUCKET_URI}"
|
||||
]
|
||||
"! gcloud storage buckets update --uniform-bucket-level-access {BUCKET_URI}" ]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
@@ -1910,9 +1907,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! gsutil -m rm -r $BUCKET_URI\n",
|
||||
"! gsutil rb $BUCKET_URI"
|
||||
]
|
||||
"! gcloud storage rm --recursive $BUCKET_URI\n", "! gcloud storage buckets delete $BUCKET_URI" ]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
|
||||
@@ -38,7 +38,7 @@
|
||||
" </td>\n",
|
||||
" <td>\n",
|
||||
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/feature_store/mobile_gaming/mobile_gaming_feature_store.ipynb\">\n",
|
||||
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
|
||||
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
|
||||
" View on GitHub\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
|
||||
@@ -37,7 +37,7 @@
|
||||
" </td>\n",
|
||||
" <td>\n",
|
||||
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/feature_store/sdk-feature-store.ipynb\">\n",
|
||||
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
|
||||
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
|
||||
" View on GitHub\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
|
||||
@@ -39,7 +39,7 @@
|
||||
" </td>\n",
|
||||
" <td>\n",
|
||||
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/gapic/automl/showcase_automl_image_classification_batch.ipynb\">\n",
|
||||
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
|
||||
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
|
||||
" View on GitHub\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
@@ -421,7 +421,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! gsutil mb -l $REGION $BUCKET_NAME"
|
||||
"! gcloud storage buckets create --location=$REGION $BUCKET_NAME"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -441,7 +441,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! gsutil ls -al $BUCKET_NAME"
|
||||
"! gcloud storage ls --all-versions --long $BUCKET_NAME"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -478,9 +478,7 @@
|
||||
"import time\n",
|
||||
"\n",
|
||||
"from google.cloud.aiplatform import gapic as aip\n",
|
||||
"from google.protobuf import json_format\n",
|
||||
"from google.protobuf.json_format import MessageToJson, ParseDict\n",
|
||||
"from google.protobuf.struct_pb2 import Struct, Value"
|
||||
"from google.protobuf import json_format"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -886,11 +884,11 @@
|
||||
"else:\n",
|
||||
" FILE = IMPORT_FILE\n",
|
||||
"\n",
|
||||
"count = ! gsutil cat $FILE | wc -l\n",
|
||||
"count = ! gcloud storage cat $FILE | wc -l\n",
|
||||
"print(\"Number of Examples\", int(count[0]))\n",
|
||||
"\n",
|
||||
"print(\"First 10 rows\")\n",
|
||||
"! gsutil cat $FILE | head"
|
||||
"! gcloud storage cat $FILE | head"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -1329,7 +1327,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"test_items = !gsutil cat $IMPORT_FILE | head -n2\n",
|
||||
"test_items = !gcloud storage cat $IMPORT_FILE | head -n2\n",
|
||||
"if len(str(test_items[0]).split(\",\")) == 3:\n",
|
||||
" _, test_item_1, test_label_1 = str(test_items[0]).split(\",\")\n",
|
||||
" _, test_item_2, test_label_2 = str(test_items[1]).split(\",\")\n",
|
||||
@@ -1363,8 +1361,8 @@
|
||||
"file_1 = test_item_1.split(\"/\")[-1]\n",
|
||||
"file_2 = test_item_2.split(\"/\")[-1]\n",
|
||||
"\n",
|
||||
"! gsutil cp $test_item_1 $BUCKET_NAME/$file_1\n",
|
||||
"! gsutil cp $test_item_2 $BUCKET_NAME/$file_2\n",
|
||||
"! gcloud storage cp $test_item_1 $BUCKET_NAME/$file_1\n",
|
||||
"! gcloud storage cp $test_item_2 $BUCKET_NAME/$file_2\n",
|
||||
"\n",
|
||||
"test_item_1 = BUCKET_NAME + \"/\" + file_1\n",
|
||||
"test_item_2 = BUCKET_NAME + \"/\" + file_2"
|
||||
@@ -1408,7 +1406,7 @@
|
||||
" f.write(json.dumps(data) + \"\\n\")\n",
|
||||
"\n",
|
||||
"print(gcs_input_uri)\n",
|
||||
"! gsutil cat $gcs_input_uri"
|
||||
"! gcloud storage cat $gcs_input_uri"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -1692,8 +1690,8 @@
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def get_latest_predictions(gcs_out_dir):\n",
|
||||
" \"\"\" Get the latest prediction subfolder using the timestamp in the subfolder name\"\"\"\n",
|
||||
" folders = !gsutil ls $gcs_out_dir\n",
|
||||
" \"\"\"Get the latest prediction subfolder using the timestamp in the subfolder name\"\"\"\n",
|
||||
" folders = !gcloud storage ls $gcs_out_dir\n",
|
||||
" latest = \"\"\n",
|
||||
" for folder in folders:\n",
|
||||
" subfolder = folder.split(\"/\")[-2]\n",
|
||||
@@ -1711,9 +1709,9 @@
|
||||
" raise Exception(\"Batch Job Failed\")\n",
|
||||
" else:\n",
|
||||
" folder = get_latest_predictions(predictions)\n",
|
||||
" ! gsutil ls $folder/prediction*.jsonl\n",
|
||||
" ! gcloud storage ls $folder/prediction*.jsonl\n",
|
||||
"\n",
|
||||
" ! gsutil cat $folder/prediction*.jsonl\n",
|
||||
" ! gcloud storage cat $folder/prediction*.jsonl\n",
|
||||
" break\n",
|
||||
" time.sleep(60)"
|
||||
]
|
||||
@@ -1808,7 +1806,7 @@
|
||||
" print(e)\n",
|
||||
"\n",
|
||||
"if delete_bucket and \"BUCKET_NAME\" in globals():\n",
|
||||
" ! gsutil rm -r $BUCKET_NAME"
|
||||
" ! gcloud storage rm --recursive $BUCKET_NAME"
|
||||
]
|
||||
}
|
||||
],
|
||||
|
||||
+9
-11
@@ -39,7 +39,7 @@
|
||||
" </td>\n",
|
||||
" <td>\n",
|
||||
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/gapic/automl/showcase_automl_image_classification_export_edge.ipynb\">\n",
|
||||
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
|
||||
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
|
||||
" View on GitHub\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
@@ -416,7 +416,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! gsutil mb -l $REGION $BUCKET_NAME"
|
||||
"! gcloud storage buckets create --location=$REGION $BUCKET_NAME"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -436,7 +436,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! gsutil ls -al $BUCKET_NAME"
|
||||
"! gcloud storage ls --all-versions --long $BUCKET_NAME"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -474,7 +474,6 @@
|
||||
"\n",
|
||||
"from google.cloud.aiplatform import gapic as aip\n",
|
||||
"from google.protobuf import json_format\n",
|
||||
"from google.protobuf.json_format import MessageToJson, ParseDict\n",
|
||||
"from google.protobuf.struct_pb2 import Struct, Value"
|
||||
]
|
||||
},
|
||||
@@ -784,11 +783,11 @@
|
||||
"else:\n",
|
||||
" FILE = IMPORT_FILE\n",
|
||||
"\n",
|
||||
"count = ! gsutil cat $FILE | wc -l\n",
|
||||
"count = ! gcloud storage cat $FILE | wc -l\n",
|
||||
"print(\"Number of Examples\", int(count[0]))\n",
|
||||
"\n",
|
||||
"print(\"First 10 rows\")\n",
|
||||
"! gsutil cat $FILE | head"
|
||||
"! gcloud storage cat $FILE | head"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -1215,7 +1214,6 @@
|
||||
" }\n",
|
||||
" response = clients[\"model\"].export_model(name=name, output_config=output_config)\n",
|
||||
" print(\"Long running operation:\", response.operation.name)\n",
|
||||
" result = response.result(timeout=1800)\n",
|
||||
" metadata = response.operation.metadata\n",
|
||||
" artifact_uri = str(metadata.value).split(\"\\\\\")[-1][4:-1]\n",
|
||||
" print(\"Artifact Uri\", artifact_uri)\n",
|
||||
@@ -1244,9 +1242,9 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! gsutil ls $model_package\n",
|
||||
"! gcloud storage ls $model_package\n",
|
||||
"# Download the model artifacts\n",
|
||||
"! gsutil cp -r $model_package tflite\n",
|
||||
"! gcloud storage cp --recursive $model_package tflite\n",
|
||||
"\n",
|
||||
"tflite_path = \"tflite/model.tflite\""
|
||||
]
|
||||
@@ -1305,7 +1303,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"test_items = ! gsutil cat $IMPORT_FILE | head -n1\n",
|
||||
"test_items = ! gcloud storage cat $IMPORT_FILE | head -n1\n",
|
||||
"test_item = test_items[0].split(\",\")[0]\n",
|
||||
"\n",
|
||||
"with tf.io.gfile.GFile(test_item, \"rb\") as f:\n",
|
||||
@@ -1449,7 +1447,7 @@
|
||||
" print(e)\n",
|
||||
"\n",
|
||||
"if delete_bucket and \"BUCKET_NAME\" in globals():\n",
|
||||
" ! gsutil rm -r $BUCKET_NAME"
|
||||
" ! gcloud storage rm --recursive $BUCKET_NAME"
|
||||
]
|
||||
}
|
||||
],
|
||||
|
||||
@@ -39,7 +39,7 @@
|
||||
" </td>\n",
|
||||
" <td>\n",
|
||||
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/gapic/automl/showcase_automl_image_classification_online.ipynb\">\n",
|
||||
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
|
||||
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
|
||||
" View on GitHub\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
@@ -721,12 +721,10 @@
|
||||
"else:\n",
|
||||
" FILE = IMPORT_FILE\n",
|
||||
"\n",
|
||||
"count = ! gsutil cat $FILE | wc -l\n",
|
||||
"print(\"Number of Examples\", int(count[0]))\n",
|
||||
"count = ! gcloud storage cat $FILE | wc -l\n", "print(\"Number of Examples\", int(count[0]))\n",
|
||||
"\n",
|
||||
"print(\"First 10 rows\")\n",
|
||||
"! gsutil cat $FILE | head"
|
||||
]
|
||||
"! gcloud storage cat $FILE | head" ]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
@@ -1342,8 +1340,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"test_item = !gsutil cat $IMPORT_FILE | head -n1\n",
|
||||
"if len(str(test_item[0]).split(\",\")) == 3:\n",
|
||||
"test_item = !gcloud storage cat $IMPORT_FILE | head -n1\n", "if len(str(test_item[0]).split(\",\")) == 3:\n",
|
||||
" _, test_item, test_label = str(test_item[0]).split(\",\")\n",
|
||||
"else:\n",
|
||||
" test_item, test_label = str(test_item[0]).split(\",\")\n",
|
||||
@@ -1568,8 +1565,7 @@
|
||||
" print(e)\n",
|
||||
"\n",
|
||||
"if delete_bucket and \"BUCKET_NAME\" in globals():\n",
|
||||
" ! gsutil rm -r $BUCKET_NAME"
|
||||
]
|
||||
" ! gcloud storage rm --recursive $BUCKET_NAME" ]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
|
||||
+5
-9
@@ -39,7 +39,7 @@
|
||||
" </td>\n",
|
||||
" <td>\n",
|
||||
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/gapic/automl/showcase_automl_image_classification_online_proxy.ipynb\">\n",
|
||||
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
|
||||
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
|
||||
" View on GitHub\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
@@ -721,12 +721,10 @@
|
||||
"else:\n",
|
||||
" FILE = IMPORT_FILE\n",
|
||||
"\n",
|
||||
"count = ! gsutil cat $FILE | wc -l\n",
|
||||
"print(\"Number of Examples\", int(count[0]))\n",
|
||||
"count = ! gcloud storage cat $FILE | wc -l\n", "print(\"Number of Examples\", int(count[0]))\n",
|
||||
"\n",
|
||||
"print(\"First 10 rows\")\n",
|
||||
"! gsutil cat $FILE | head"
|
||||
]
|
||||
"! gcloud storage cat $FILE | head" ]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
@@ -1342,8 +1340,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"test_item = !gsutil cat $IMPORT_FILE | head -n1\n",
|
||||
"if len(str(test_item[0]).split(\",\")) == 3:\n",
|
||||
"test_item = !gcloud storage cat $IMPORT_FILE | head -n1\n", "if len(str(test_item[0]).split(\",\")) == 3:\n",
|
||||
" _, test_item, test_label = str(test_item[0]).split(\",\")\n",
|
||||
"else:\n",
|
||||
" test_item, test_label = str(test_item[0]).split(\",\")\n",
|
||||
@@ -1747,8 +1744,7 @@
|
||||
" print(e)\n",
|
||||
"\n",
|
||||
"if delete_bucket and \"BUCKET_NAME\" in globals():\n",
|
||||
" ! gsutil rm -r $BUCKET_NAME"
|
||||
]
|
||||
" ! gcloud storage rm --recursive $BUCKET_NAME" ]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
|
||||
+15
-17
@@ -39,7 +39,7 @@
|
||||
" </td>\n",
|
||||
" <td>\n",
|
||||
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/gapic/automl/showcase_automl_image_object_detection_batch.ipynb\">\n",
|
||||
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
|
||||
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
|
||||
" View on GitHub\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
@@ -421,7 +421,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! gsutil mb -l $REGION $BUCKET_NAME"
|
||||
"! gcloud storage buckets create --location=$REGION $BUCKET_NAME"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -441,7 +441,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! gsutil ls -al $BUCKET_NAME"
|
||||
"! gcloud storage ls --all-versions --long $BUCKET_NAME"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -478,9 +478,7 @@
|
||||
"import time\n",
|
||||
"\n",
|
||||
"from google.cloud.aiplatform import gapic as aip\n",
|
||||
"from google.protobuf import json_format\n",
|
||||
"from google.protobuf.json_format import MessageToJson, ParseDict\n",
|
||||
"from google.protobuf.struct_pb2 import Struct, Value"
|
||||
"from google.protobuf import json_format"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -887,11 +885,11 @@
|
||||
"else:\n",
|
||||
" FILE = IMPORT_FILE\n",
|
||||
"\n",
|
||||
"count = ! gsutil cat $FILE | wc -l\n",
|
||||
"count = ! gcloud storage cat $FILE | wc -l\n",
|
||||
"print(\"Number of Examples\", int(count[0]))\n",
|
||||
"\n",
|
||||
"print(\"First 10 rows\")\n",
|
||||
"! gsutil cat $FILE | head"
|
||||
"! gcloud storage cat $FILE | head"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -1333,7 +1331,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"test_items = !gsutil cat $IMPORT_FILE | head -n2\n",
|
||||
"test_items = !gcloud storage cat $IMPORT_FILE | head -n2\n",
|
||||
"cols_1 = str(test_items[0]).split(\",\")\n",
|
||||
"cols_2 = str(test_items[1]).split(\",\")\n",
|
||||
"if len(cols_1) == 11:\n",
|
||||
@@ -1373,8 +1371,8 @@
|
||||
"file_1 = test_item_1.split(\"/\")[-1]\n",
|
||||
"file_2 = test_item_2.split(\"/\")[-1]\n",
|
||||
"\n",
|
||||
"! gsutil cp $test_item_1 $BUCKET_NAME/$file_1\n",
|
||||
"! gsutil cp $test_item_2 $BUCKET_NAME/$file_2\n",
|
||||
"! gcloud storage cp $test_item_1 $BUCKET_NAME/$file_1\n",
|
||||
"! gcloud storage cp $test_item_2 $BUCKET_NAME/$file_2\n",
|
||||
"\n",
|
||||
"test_item_1 = BUCKET_NAME + \"/\" + file_1\n",
|
||||
"test_item_2 = BUCKET_NAME + \"/\" + file_2"
|
||||
@@ -1418,7 +1416,7 @@
|
||||
" f.write(json.dumps(data) + \"\\n\")\n",
|
||||
"\n",
|
||||
"print(gcs_input_uri)\n",
|
||||
"! gsutil cat $gcs_input_uri"
|
||||
"! gcloud storage cat $gcs_input_uri"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -1704,8 +1702,8 @@
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def get_latest_predictions(gcs_out_dir):\n",
|
||||
" \"\"\" Get the latest prediction subfolder using the timestamp in the subfolder name\"\"\"\n",
|
||||
" folders = !gsutil ls $gcs_out_dir\n",
|
||||
" \"\"\"Get the latest prediction subfolder using the timestamp in the subfolder name\"\"\"\n",
|
||||
" folders = !gcloud storage ls $gcs_out_dir\n",
|
||||
" latest = \"\"\n",
|
||||
" for folder in folders:\n",
|
||||
" subfolder = folder.split(\"/\")[-2]\n",
|
||||
@@ -1723,9 +1721,9 @@
|
||||
" raise Exception(\"Batch Job Failed\")\n",
|
||||
" else:\n",
|
||||
" folder = get_latest_predictions(predictions)\n",
|
||||
" ! gsutil ls $folder/prediction*.jsonl\n",
|
||||
" ! gcloud storage ls $folder/prediction*.jsonl\n",
|
||||
"\n",
|
||||
" ! gsutil cat $folder/prediction*.jsonl\n",
|
||||
" ! gcloud storage cat $folder/prediction*.jsonl\n",
|
||||
" break\n",
|
||||
" time.sleep(60)"
|
||||
]
|
||||
@@ -1820,7 +1818,7 @@
|
||||
" print(e)\n",
|
||||
"\n",
|
||||
"if delete_bucket and \"BUCKET_NAME\" in globals():\n",
|
||||
" ! gsutil rm -r $BUCKET_NAME"
|
||||
" ! gcloud storage rm --recursive $BUCKET_NAME"
|
||||
]
|
||||
}
|
||||
],
|
||||
|
||||
+10
-13
@@ -39,7 +39,7 @@
|
||||
" </td>\n",
|
||||
" <td>\n",
|
||||
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/gapic/automl/showcase_automl_image_object_detection_export_edge.ipynb\">\n",
|
||||
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
|
||||
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
|
||||
" View on GitHub\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
@@ -416,7 +416,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! gsutil mb -l $REGION $BUCKET_NAME"
|
||||
"! gcloud storage buckets create --location $REGION $BUCKET_NAME"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -436,7 +436,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! gsutil ls -al $BUCKET_NAME"
|
||||
"! gcloud storage ls --all-versions --long $BUCKET_NAME"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -473,9 +473,7 @@
|
||||
"import time\n",
|
||||
"\n",
|
||||
"from google.cloud.aiplatform import gapic as aip\n",
|
||||
"from google.protobuf import json_format\n",
|
||||
"from google.protobuf.json_format import MessageToJson, ParseDict\n",
|
||||
"from google.protobuf.struct_pb2 import Struct, Value"
|
||||
"from google.protobuf import json_format\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -785,11 +783,11 @@
|
||||
"else:\n",
|
||||
" FILE = IMPORT_FILE\n",
|
||||
"\n",
|
||||
"count = ! gsutil cat $FILE | wc -l\n",
|
||||
"count = ! gcloud storage cat $FILE | wc -l\n",
|
||||
"print(\"Number of Examples\", int(count[0]))\n",
|
||||
"\n",
|
||||
"print(\"First 10 rows\")\n",
|
||||
"! gsutil cat $FILE | head"
|
||||
"! gcloud storage cat $FILE | head"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -1218,7 +1216,6 @@
|
||||
" }\n",
|
||||
" response = clients[\"model\"].export_model(name=name, output_config=output_config)\n",
|
||||
" print(\"Long running operation:\", response.operation.name)\n",
|
||||
" result = response.result(timeout=1800)\n",
|
||||
" metadata = response.operation.metadata\n",
|
||||
" artifact_uri = str(metadata.value).split(\"\\\\\")[-1][4:-1]\n",
|
||||
" print(\"Artifact Uri\", artifact_uri)\n",
|
||||
@@ -1247,9 +1244,9 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! gsutil ls $model_package\n",
|
||||
"! gcloud storage ls $model_package\n",
|
||||
"# Download the model artifacts\n",
|
||||
"! gsutil cp -r $model_package tflite\n",
|
||||
"! gcloud storage cp --recursive $model_package tflite\n",
|
||||
"\n",
|
||||
"tflite_path = \"tflite/model.tflite\""
|
||||
]
|
||||
@@ -1308,7 +1305,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"test_items = ! gsutil cat $IMPORT_FILE | head -n1\n",
|
||||
"test_items = ! gcloud storage cat $IMPORT_FILE | head -n1\n",
|
||||
"test_item = test_items[0].split(\",\")[0]\n",
|
||||
"\n",
|
||||
"with tf.io.gfile.GFile(test_item, \"rb\") as f:\n",
|
||||
@@ -1452,7 +1449,7 @@
|
||||
" print(e)\n",
|
||||
"\n",
|
||||
"if delete_bucket and \"BUCKET_NAME\" in globals():\n",
|
||||
" ! gsutil rm -r $BUCKET_NAME"
|
||||
" ! gcloud storage rm --recursive $BUCKET_NAME"
|
||||
]
|
||||
}
|
||||
],
|
||||
|
||||
+6
-8
@@ -39,7 +39,7 @@
|
||||
" </td>\n",
|
||||
" <td>\n",
|
||||
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/gapic/automl/showcase_automl_image_object_detection_online.ipynb\">\n",
|
||||
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
|
||||
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
|
||||
" View on GitHub\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
@@ -396,9 +396,7 @@
|
||||
"import time\n",
|
||||
"\n",
|
||||
"from google.cloud.aiplatform import gapic as aip\n",
|
||||
"from google.protobuf import json_format\n",
|
||||
"from google.protobuf.json_format import MessageToJson, ParseDict\n",
|
||||
"from google.protobuf.struct_pb2 import Struct, Value"
|
||||
"from google.protobuf import json_format"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -722,11 +720,11 @@
|
||||
"else:\n",
|
||||
" FILE = IMPORT_FILE\n",
|
||||
"\n",
|
||||
"count = ! gsutil cat $FILE | wc -l\n",
|
||||
"count = ! gcloud storage cat $FILE | wc -l\n",
|
||||
"print(\"Number of Examples\", int(count[0]))\n",
|
||||
"\n",
|
||||
"print(\"First 10 rows\")\n",
|
||||
"! gsutil cat $FILE | head"
|
||||
"! gcloud storage cat $FILE | head"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -1345,7 +1343,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"test_items = !gsutil cat $IMPORT_FILE | head -n1\n",
|
||||
"test_items = !gcloud storage cat $IMPORT_FILE | head -n1\n",
|
||||
"cols = str(test_items[0]).split(\",\")\n",
|
||||
"if len(cols) == 11:\n",
|
||||
" test_item = str(cols[1])\n",
|
||||
@@ -1574,7 +1572,7 @@
|
||||
" print(e)\n",
|
||||
"\n",
|
||||
"if delete_bucket and \"BUCKET_NAME\" in globals():\n",
|
||||
" ! gsutil rm -r $BUCKET_NAME"
|
||||
" ! gcloud storage rm --recursive $BUCKET_NAME"
|
||||
]
|
||||
}
|
||||
],
|
||||
|
||||
@@ -39,7 +39,7 @@
|
||||
" </td>\n",
|
||||
" <td>\n",
|
||||
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/gapic/automl/showcase_automl_image_segmentation_batch.ipynb\">\n",
|
||||
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
|
||||
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
|
||||
" View on GitHub\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
@@ -421,8 +421,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! gsutil mb -l $REGION $BUCKET_NAME"
|
||||
]
|
||||
"! gcloud storage buckets create --location $REGION $BUCKET_NAME" ]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
@@ -441,8 +440,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! gsutil ls -al $BUCKET_NAME"
|
||||
]
|
||||
"! gcloud storage ls --all-versions --long $BUCKET_NAME" ]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
@@ -891,12 +889,10 @@
|
||||
"else:\n",
|
||||
" FILE = IMPORT_FILE\n",
|
||||
"\n",
|
||||
"count = ! gsutil cat $FILE | wc -l\n",
|
||||
"print(\"Number of Examples\", int(count[0]))\n",
|
||||
"count = ! gcloud storage cat $FILE | wc -l\n", "print(\"Number of Examples\", int(count[0]))\n",
|
||||
"\n",
|
||||
"print(\"First 10 rows\")\n",
|
||||
"! gsutil cat $FILE | head"
|
||||
]
|
||||
"! gcloud storage cat $FILE | head" ]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
@@ -1325,8 +1321,7 @@
|
||||
"source": [
|
||||
"import json\n",
|
||||
"\n",
|
||||
"test_items = !gsutil cat $IMPORT_FILE | head -n2\n",
|
||||
"test_data_1 = test_items[0].replace(\"'\", '\"')\n",
|
||||
"test_items = !gcloud storage cat $IMPORT_FILE | head -n2\n", "test_data_1 = test_items[0].replace(\"'\", '\"')\n",
|
||||
"test_data_1 = json.loads(test_data_1)\n",
|
||||
"test_data_2 = test_items[0].replace(\"'\", '\"')\n",
|
||||
"test_data_2 = json.loads(test_data_2)\n",
|
||||
@@ -1367,9 +1362,8 @@
|
||||
"file_1 = test_item_1.split(\"/\")[-1]\n",
|
||||
"file_2 = test_item_2.split(\"/\")[-1]\n",
|
||||
"\n",
|
||||
"! gsutil cp $test_item_1 $BUCKET_NAME/$file_1\n",
|
||||
"! gsutil cp $test_item_2 $BUCKET_NAME/$file_2\n",
|
||||
"\n",
|
||||
"! gcloud storage cp $test_item_1 $BUCKET_NAME/$file_1\n",
|
||||
"! gcloud storage cp $test_item_2 $BUCKET_NAME/$file_2\n", "\n",
|
||||
"test_item_1 = BUCKET_NAME + \"/\" + file_1\n",
|
||||
"test_item_2 = BUCKET_NAME + \"/\" + file_2"
|
||||
]
|
||||
@@ -1412,8 +1406,7 @@
|
||||
" f.write(json.dumps(data) + \"\\n\")\n",
|
||||
"\n",
|
||||
"print(gcs_input_uri)\n",
|
||||
"! gsutil cat $gcs_input_uri"
|
||||
]
|
||||
"! gcloud storage cat $gcs_input_uri" ]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
@@ -1681,8 +1674,7 @@
|
||||
"source": [
|
||||
"def get_latest_predictions(gcs_out_dir):\n",
|
||||
" \"\"\" Get the latest prediction subfolder using the timestamp in the subfolder name\"\"\"\n",
|
||||
" folders = !gsutil ls $gcs_out_dir\n",
|
||||
" latest = \"\"\n",
|
||||
" folders = !gcloud storage ls $gcs_out_dir\n", " latest = \"\"\n",
|
||||
" for folder in folders:\n",
|
||||
" subfolder = folder.split(\"/\")[-2]\n",
|
||||
" if subfolder.startswith(\"prediction-\"):\n",
|
||||
@@ -1699,10 +1691,8 @@
|
||||
" raise Exception(\"Batch Job Failed\")\n",
|
||||
" else:\n",
|
||||
" folder = get_latest_predictions(predictions)\n",
|
||||
" ! gsutil ls $folder/prediction*.jsonl\n",
|
||||
"\n",
|
||||
" ! gsutil cat $folder/prediction*.jsonl\n",
|
||||
" break\n",
|
||||
" ! gcloud storage ls $folder/prediction*.jsonl\n", "\n",
|
||||
" ! gcloud storage cat $folder/prediction*.jsonl\n", " break\n",
|
||||
" time.sleep(60)"
|
||||
]
|
||||
},
|
||||
@@ -1796,8 +1786,7 @@
|
||||
" print(e)\n",
|
||||
"\n",
|
||||
"if delete_bucket and \"BUCKET_NAME\" in globals():\n",
|
||||
" ! gsutil rm -r $BUCKET_NAME"
|
||||
]
|
||||
" ! gcloud storage rm --recursive $BUCKET_NAME" ]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
|
||||
@@ -39,7 +39,7 @@
|
||||
" </td>\n",
|
||||
" <td>\n",
|
||||
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/gapic/automl/showcase_automl_image_segmentation_online.ipynb\">\n",
|
||||
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
|
||||
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
|
||||
" View on GitHub\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
@@ -726,12 +726,10 @@
|
||||
"else:\n",
|
||||
" FILE = IMPORT_FILE\n",
|
||||
"\n",
|
||||
"count = ! gsutil cat $FILE | wc -l\n",
|
||||
"print(\"Number of Examples\", int(count[0]))\n",
|
||||
"count = ! gcloud storage cat $FILE | wc -l\n", "print(\"Number of Examples\", int(count[0]))\n",
|
||||
"\n",
|
||||
"print(\"First 10 rows\")\n",
|
||||
"! gsutil cat $FILE | head"
|
||||
]
|
||||
"! gcloud storage cat $FILE | head" ]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
@@ -1338,8 +1336,7 @@
|
||||
"source": [
|
||||
"import json\n",
|
||||
"\n",
|
||||
"test_items = !gsutil cat $IMPORT_FILE | head -n1\n",
|
||||
"test_data = test_items[0].replace(\"'\", '\"')\n",
|
||||
"test_items = !gcloud storage cat $IMPORT_FILE | head -n1\n", "test_data = test_items[0].replace(\"'\", '\"')\n",
|
||||
"test_data = json.loads(test_data)\n",
|
||||
"try:\n",
|
||||
" test_item = test_data[\"image_gcs_uri\"]\n",
|
||||
@@ -1555,8 +1552,7 @@
|
||||
" print(e)\n",
|
||||
"\n",
|
||||
"if delete_bucket and \"BUCKET_NAME\" in globals():\n",
|
||||
" ! gsutil rm -r $BUCKET_NAME"
|
||||
]
|
||||
" ! gcloud storage rm --recursive $BUCKET_NAME" ]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
|
||||
+11
-21
@@ -39,7 +39,7 @@
|
||||
" </td>\n",
|
||||
" <td>\n",
|
||||
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/gapic/automl/showcase_automl_tabular_binary_classification_batch.ipynb\">\n",
|
||||
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
|
||||
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
|
||||
" View on GitHub\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
@@ -421,8 +421,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! gsutil mb -l $REGION $BUCKET_NAME"
|
||||
]
|
||||
"! gcloud storage buckets create --location $REGION $BUCKET_NAME" ]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
@@ -441,8 +440,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! gsutil ls -al $BUCKET_NAME"
|
||||
]
|
||||
"! gcloud storage ls --all-versions --long $BUCKET_NAME" ]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
@@ -809,14 +807,11 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"count = ! gsutil cat $IMPORT_FILE | wc -l\n",
|
||||
"print(\"Number of Examples\", int(count[0]))\n",
|
||||
"count = ! gcloud storage cat $IMPORT_FILE | wc -l\n", "print(\"Number of Examples\", int(count[0]))\n",
|
||||
"\n",
|
||||
"print(\"First 10 rows\")\n",
|
||||
"! gsutil cat $IMPORT_FILE | head\n",
|
||||
"\n",
|
||||
"heading = ! gsutil cat $IMPORT_FILE | head -n1\n",
|
||||
"label_column = str(heading).split(\",\")[-1].split(\"'\")[0]\n",
|
||||
"! gcloud storage cat $IMPORT_FILE | head\n", "\n",
|
||||
"heading = ! gcloud storage cat $IMPORT_FILE | head -n1\n", "label_column = str(heading).split(\",\")[-1].split(\"'\")[0]\n",
|
||||
"print(\"Label Column Name\", label_column)\n",
|
||||
"if label_column is None:\n",
|
||||
" raise Exception(\"label column missing\")"
|
||||
@@ -1377,8 +1372,7 @@
|
||||
" f.write(str(INSTANCE_2) + \"\\n\")\n",
|
||||
"\n",
|
||||
"print(gcs_input_uri)\n",
|
||||
"! gsutil cat $gcs_input_uri"
|
||||
]
|
||||
"! gcloud storage cat $gcs_input_uri" ]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
@@ -1646,8 +1640,7 @@
|
||||
"source": [
|
||||
"def get_latest_predictions(gcs_out_dir):\n",
|
||||
" \"\"\" Get the latest prediction subfolder using the timestamp in the subfolder name\"\"\"\n",
|
||||
" folders = !gsutil ls $gcs_out_dir\n",
|
||||
" latest = \"\"\n",
|
||||
" folders = !gcloud storage ls $gcs_out_dir\n", " latest = \"\"\n",
|
||||
" for folder in folders:\n",
|
||||
" subfolder = folder.split(\"/\")[-2]\n",
|
||||
" if subfolder.startswith(\"prediction-\"):\n",
|
||||
@@ -1664,10 +1657,8 @@
|
||||
" raise Exception(\"Batch Job Failed\")\n",
|
||||
" else:\n",
|
||||
" folder = get_latest_predictions(predictions)\n",
|
||||
" ! gsutil ls $folder/prediction*.csv\n",
|
||||
"\n",
|
||||
" ! gsutil cat $folder/prediction*.csv\n",
|
||||
" break\n",
|
||||
" ! gcloud storage ls $folder/prediction*.csv\n", "\n",
|
||||
" ! gcloud storage cat $folder/prediction*.csv\n", " break\n",
|
||||
" time.sleep(60)"
|
||||
]
|
||||
},
|
||||
@@ -1761,8 +1752,7 @@
|
||||
" print(e)\n",
|
||||
"\n",
|
||||
"if delete_bucket and \"BUCKET_NAME\" in globals():\n",
|
||||
" ! gsutil rm -r $BUCKET_NAME"
|
||||
]
|
||||
" ! gcloud storage rm --recursive $BUCKET_NAME" ]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
|
||||
+5
-9
@@ -39,7 +39,7 @@
|
||||
" </td>\n",
|
||||
" <td>\n",
|
||||
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/gapic/automl/showcase_automl_tabular_binary_classification_online.ipynb\">\n",
|
||||
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
|
||||
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
|
||||
" View on GitHub\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
@@ -735,14 +735,11 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"count = ! gsutil cat $IMPORT_FILE | wc -l\n",
|
||||
"print(\"Number of Examples\", int(count[0]))\n",
|
||||
"count = ! gcloud storage cat $IMPORT_FILE | wc -l\n", "print(\"Number of Examples\", int(count[0]))\n",
|
||||
"\n",
|
||||
"print(\"First 10 rows\")\n",
|
||||
"! gsutil cat $IMPORT_FILE | head\n",
|
||||
"\n",
|
||||
"heading = ! gsutil cat $IMPORT_FILE | head -n1\n",
|
||||
"label_column = str(heading).split(\",\")[-1].split(\"'\")[0]\n",
|
||||
"! gcloud storage cat $IMPORT_FILE | head\n", "\n",
|
||||
"heading = ! gcloud storage cat $IMPORT_FILE | head -n1\n", "label_column = str(heading).split(\",\")[-1].split(\"'\")[0]\n",
|
||||
"print(\"Label Column Name\", label_column)\n",
|
||||
"if label_column is None:\n",
|
||||
" raise Exception(\"label column missing\")"
|
||||
@@ -1668,8 +1665,7 @@
|
||||
" print(e)\n",
|
||||
"\n",
|
||||
"if delete_bucket and \"BUCKET_NAME\" in globals():\n",
|
||||
" ! gsutil rm -r $BUCKET_NAME"
|
||||
]
|
||||
" ! gcloud storage rm --recursive $BUCKET_NAME" ]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
|
||||
+11
-21
@@ -39,7 +39,7 @@
|
||||
" </td>\n",
|
||||
" <td>\n",
|
||||
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/gapic/automl/showcase_automl_tabular_classification_batch.ipynb\">\n",
|
||||
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
|
||||
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
|
||||
" View on GitHub\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
@@ -421,8 +421,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! gsutil mb -l $REGION $BUCKET_NAME"
|
||||
]
|
||||
"! gcloud storage buckets create --location $REGION $BUCKET_NAME" ]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
@@ -441,8 +440,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! gsutil ls -al $BUCKET_NAME"
|
||||
]
|
||||
"! gcloud storage ls --all-versions --long $BUCKET_NAME" ]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
@@ -810,14 +808,11 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"count = ! gsutil cat $IMPORT_FILE | wc -l\n",
|
||||
"print(\"Number of Examples\", int(count[0]))\n",
|
||||
"count = ! gcloud storage cat $IMPORT_FILE | wc -l\n", "print(\"Number of Examples\", int(count[0]))\n",
|
||||
"\n",
|
||||
"print(\"First 10 rows\")\n",
|
||||
"! gsutil cat $IMPORT_FILE | head\n",
|
||||
"\n",
|
||||
"heading = ! gsutil cat $IMPORT_FILE | head -n1\n",
|
||||
"label_column = str(heading).split(\",\")[-1].split(\"'\")[0]\n",
|
||||
"! gcloud storage cat $IMPORT_FILE | head\n", "\n",
|
||||
"heading = ! gcloud storage cat $IMPORT_FILE | head -n1\n", "label_column = str(heading).split(\",\")[-1].split(\"'\")[0]\n",
|
||||
"print(\"Label Column Name\", label_column)\n",
|
||||
"if label_column is None:\n",
|
||||
" raise Exception(\"label column missing\")"
|
||||
@@ -1363,8 +1358,7 @@
|
||||
" f.write(str(INSTANCE_2) + \"\\n\")\n",
|
||||
"\n",
|
||||
"print(gcs_input_uri)\n",
|
||||
"! gsutil cat $gcs_input_uri"
|
||||
]
|
||||
"! gcloud storage cat $gcs_input_uri" ]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
@@ -1632,8 +1626,7 @@
|
||||
"source": [
|
||||
"def get_latest_predictions(gcs_out_dir):\n",
|
||||
" \"\"\" Get the latest prediction subfolder using the timestamp in the subfolder name\"\"\"\n",
|
||||
" folders = !gsutil ls $gcs_out_dir\n",
|
||||
" latest = \"\"\n",
|
||||
" folders = !gcloud storage ls $gcs_out_dir\n", " latest = \"\"\n",
|
||||
" for folder in folders:\n",
|
||||
" subfolder = folder.split(\"/\")[-2]\n",
|
||||
" if subfolder.startswith(\"prediction-\"):\n",
|
||||
@@ -1650,10 +1643,8 @@
|
||||
" raise Exception(\"Batch Job Failed\")\n",
|
||||
" else:\n",
|
||||
" folder = get_latest_predictions(predictions)\n",
|
||||
" ! gsutil ls $folder/prediction*.csv\n",
|
||||
"\n",
|
||||
" ! gsutil cat $folder/prediction*.csv\n",
|
||||
" break\n",
|
||||
" ! gcloud storage ls $folder/prediction*.csv\n", "\n",
|
||||
" ! gcloud storage cat $folder/prediction*.csv\n", " break\n",
|
||||
" time.sleep(60)"
|
||||
]
|
||||
},
|
||||
@@ -1747,8 +1738,7 @@
|
||||
" print(e)\n",
|
||||
"\n",
|
||||
"if delete_bucket and \"BUCKET_NAME\" in globals():\n",
|
||||
" ! gsutil rm -r $BUCKET_NAME"
|
||||
]
|
||||
" ! gcloud storage rm --recursive $BUCKET_NAME" ]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
|
||||
+11
-21
@@ -39,7 +39,7 @@
|
||||
" </td>\n",
|
||||
" <td>\n",
|
||||
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/gapic/automl/showcase_automl_tabular_classification_batch_explain.ipynb\">\n",
|
||||
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
|
||||
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
|
||||
" View on GitHub\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
@@ -421,8 +421,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! gsutil mb -l $REGION $BUCKET_NAME"
|
||||
]
|
||||
"! gcloud storage buckets create --location $REGION $BUCKET_NAME" ]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
@@ -441,8 +440,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! gsutil ls -al $BUCKET_NAME"
|
||||
]
|
||||
"! gcloud storage ls --all-versions --long $BUCKET_NAME" ]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
@@ -810,14 +808,11 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"count = ! gsutil cat $IMPORT_FILE | wc -l\n",
|
||||
"print(\"Number of Examples\", int(count[0]))\n",
|
||||
"count = ! gcloud storage cat $IMPORT_FILE | wc -l\n", "print(\"Number of Examples\", int(count[0]))\n",
|
||||
"\n",
|
||||
"print(\"First 10 rows\")\n",
|
||||
"! gsutil cat $IMPORT_FILE | head\n",
|
||||
"\n",
|
||||
"heading = ! gsutil cat $IMPORT_FILE | head -n1\n",
|
||||
"label_column = str(heading).split(\",\")[-1].split(\"'\")[0]\n",
|
||||
"! gcloud storage cat $IMPORT_FILE | head\n", "\n",
|
||||
"heading = ! gcloud storage cat $IMPORT_FILE | head -n1\n", "label_column = str(heading).split(\",\")[-1].split(\"'\")[0]\n",
|
||||
"print(\"Label Column Name\", label_column)\n",
|
||||
"if label_column is None:\n",
|
||||
" raise Exception(\"label column missing\")"
|
||||
@@ -1363,8 +1358,7 @@
|
||||
" f.write(str(INSTANCE_2) + \"\\n\")\n",
|
||||
"\n",
|
||||
"print(gcs_input_uri)\n",
|
||||
"! gsutil cat $gcs_input_uri"
|
||||
]
|
||||
"! gcloud storage cat $gcs_input_uri" ]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
@@ -1631,8 +1625,7 @@
|
||||
"source": [
|
||||
"def get_latest_predictions(gcs_out_dir):\n",
|
||||
" \"\"\" Get the latest prediction subfolder using the timestamp in the subfolder name\"\"\"\n",
|
||||
" folders = !gsutil ls $gcs_out_dir\n",
|
||||
" latest = \"\"\n",
|
||||
" folders = !gcloud storage ls $gcs_out_dir\n", " latest = \"\"\n",
|
||||
" for folder in folders:\n",
|
||||
" subfolder = folder.split(\"/\")[-2]\n",
|
||||
" if subfolder.startswith(\"prediction-\"):\n",
|
||||
@@ -1649,10 +1642,8 @@
|
||||
" raise Exception(\"Batch Job Failed\")\n",
|
||||
" else:\n",
|
||||
" folder = get_latest_predictions(predictions)\n",
|
||||
" ! gsutil ls $folder/explanation*.csv\n",
|
||||
"\n",
|
||||
" ! gsutil cat $folder/explanation*.csv\n",
|
||||
" break\n",
|
||||
" ! gcloud storage ls $folder/explanation*.csv\n", "\n",
|
||||
" ! gcloud storage cat $folder/explanation*.csv\n", " break\n",
|
||||
" time.sleep(60)"
|
||||
]
|
||||
},
|
||||
@@ -1746,8 +1737,7 @@
|
||||
" print(e)\n",
|
||||
"\n",
|
||||
"if delete_bucket and \"BUCKET_NAME\" in globals():\n",
|
||||
" ! gsutil rm -r $BUCKET_NAME"
|
||||
]
|
||||
" ! gcloud storage rm --recursive $BUCKET_NAME" ]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
|
||||
+11
-21
@@ -39,7 +39,7 @@
|
||||
" </td>\n",
|
||||
" <td>\n",
|
||||
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/gapic/automl/showcase_automl_tabular_classification_export_cloud.ipynb\">\n",
|
||||
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
|
||||
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
|
||||
" View on GitHub\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
@@ -416,8 +416,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! gsutil mb -l $REGION $BUCKET_NAME"
|
||||
]
|
||||
"! gcloud storage buckets create --location $REGION $BUCKET_NAME" ]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
@@ -436,8 +435,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! gsutil ls -al $BUCKET_NAME"
|
||||
]
|
||||
"! gcloud storage ls --all-versions --long $BUCKET_NAME" ]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
@@ -749,14 +747,11 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"count = ! gsutil cat $IMPORT_FILE | wc -l\n",
|
||||
"print(\"Number of Examples\", int(count[0]))\n",
|
||||
"count = ! gcloud storage cat $IMPORT_FILE | wc -l\n", "print(\"Number of Examples\", int(count[0]))\n",
|
||||
"\n",
|
||||
"print(\"First 10 rows\")\n",
|
||||
"! gsutil cat $IMPORT_FILE | head\n",
|
||||
"\n",
|
||||
"heading = ! gsutil cat $IMPORT_FILE | head -n1\n",
|
||||
"label_column = str(heading).split(\",\")[-1].split(\"'\")[0]\n",
|
||||
"! gcloud storage cat $IMPORT_FILE | head\n", "\n",
|
||||
"heading = ! gcloud storage cat $IMPORT_FILE | head -n1\n", "label_column = str(heading).split(\",\")[-1].split(\"'\")[0]\n",
|
||||
"print(\"Label Column Name\", label_column)\n",
|
||||
"if label_column is None:\n",
|
||||
" raise Exception(\"label column missing\")"
|
||||
@@ -1283,15 +1278,12 @@
|
||||
"source": [
|
||||
"print(\"Model Package:\", model_package)\n",
|
||||
"print(\"Contents:\")\n",
|
||||
"! gsutil ls $model_package\n",
|
||||
"\n",
|
||||
"! gcloud storage ls $model_package\n", "\n",
|
||||
"print(\"\\nTF Saved Model\")\n",
|
||||
"path = model_package + \"/predict\"\n",
|
||||
"files = ! gsutil ls $path\n",
|
||||
"saved_dir = files[1]\n",
|
||||
"files = ! gcloud storage ls $path\n", "saved_dir = files[1]\n",
|
||||
"print(saved_dir)\n",
|
||||
"! gsutil ls $saved_dir"
|
||||
]
|
||||
"! gcloud storage ls $saved_dir" ]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
@@ -1312,8 +1304,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! gsutil cp -r $model_package ."
|
||||
]
|
||||
"! gcloud storage cp --recursive $model_package ." ]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
@@ -1599,8 +1590,7 @@
|
||||
" print(e)\n",
|
||||
"\n",
|
||||
"if delete_bucket and \"BUCKET_NAME\" in globals():\n",
|
||||
" ! gsutil rm -r $BUCKET_NAME"
|
||||
]
|
||||
" ! gcloud storage rm --recursive $BUCKET_NAME" ]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
|
||||
+5
-9
@@ -39,7 +39,7 @@
|
||||
" </td>\n",
|
||||
" <td>\n",
|
||||
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/gapic/automl/showcase_automl_tabular_classification_online.ipynb\">\n",
|
||||
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
|
||||
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
|
||||
" View on GitHub\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
@@ -735,14 +735,11 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"count = ! gsutil cat $IMPORT_FILE | wc -l\n",
|
||||
"print(\"Number of Examples\", int(count[0]))\n",
|
||||
"count = ! gcloud storage cat $IMPORT_FILE | wc -l\n", "print(\"Number of Examples\", int(count[0]))\n",
|
||||
"\n",
|
||||
"print(\"First 10 rows\")\n",
|
||||
"! gsutil cat $IMPORT_FILE | head\n",
|
||||
"\n",
|
||||
"heading = ! gsutil cat $IMPORT_FILE | head -n1\n",
|
||||
"label_column = str(heading).split(\",\")[-1].split(\"'\")[0]\n",
|
||||
"! gcloud storage cat $IMPORT_FILE | head\n", "\n",
|
||||
"heading = ! gcloud storage cat $IMPORT_FILE | head -n1\n", "label_column = str(heading).split(\",\")[-1].split(\"'\")[0]\n",
|
||||
"print(\"Label Column Name\", label_column)\n",
|
||||
"if label_column is None:\n",
|
||||
" raise Exception(\"label column missing\")"
|
||||
@@ -1645,8 +1642,7 @@
|
||||
" print(e)\n",
|
||||
"\n",
|
||||
"if delete_bucket and \"BUCKET_NAME\" in globals():\n",
|
||||
" ! gsutil rm -r $BUCKET_NAME"
|
||||
]
|
||||
" ! gcloud storage rm --recursive $BUCKET_NAME" ]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
|
||||
+5
-6
@@ -39,7 +39,7 @@
|
||||
" </td>\n",
|
||||
" <td>\n",
|
||||
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/gapic/automl/showcase_automl_tabular_classification_online_explain.ipynb\">\n",
|
||||
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
|
||||
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
|
||||
" View on GitHub\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
@@ -397,7 +397,6 @@
|
||||
"\n",
|
||||
"import google.cloud.aiplatform_v1beta1 as aip\n",
|
||||
"from google.protobuf import json_format\n",
|
||||
"from google.protobuf.json_format import MessageToJson, ParseDict\n",
|
||||
"from google.protobuf.struct_pb2 import Struct, Value"
|
||||
]
|
||||
},
|
||||
@@ -735,13 +734,13 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"count = ! gsutil cat $IMPORT_FILE | wc -l\n",
|
||||
"count = ! gcloud storage cat $IMPORT_FILE | wc -l\n",
|
||||
"print(\"Number of Examples\", int(count[0]))\n",
|
||||
"\n",
|
||||
"print(\"First 10 rows\")\n",
|
||||
"! gsutil cat $IMPORT_FILE | head\n",
|
||||
"! gcloud storage cat $IMPORT_FILE | head\n",
|
||||
"\n",
|
||||
"heading = ! gsutil cat $IMPORT_FILE | head -n1\n",
|
||||
"heading = ! gcloud storage cat $IMPORT_FILE | head -n1\n",
|
||||
"label_column = str(heading).split(\",\")[-1].split(\"'\")[0]\n",
|
||||
"print(\"Label Column Name\", label_column)\n",
|
||||
"if label_column is None:\n",
|
||||
@@ -1820,7 +1819,7 @@
|
||||
" print(e)\n",
|
||||
"\n",
|
||||
"if delete_bucket and \"BUCKET_NAME\" in globals():\n",
|
||||
" ! gsutil rm -r $BUCKET_NAME"
|
||||
" ! gcloud storage rm --recursive $BUCKET_NAME"
|
||||
]
|
||||
}
|
||||
],
|
||||
|
||||
@@ -39,7 +39,7 @@
|
||||
" </td>\n",
|
||||
" <td>\n",
|
||||
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/gapic/automl/showcase_automl_tabular_forecasting_batch.ipynb\">\n",
|
||||
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
|
||||
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
|
||||
" View on GitHub\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
@@ -421,8 +421,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! gsutil mb -l $REGION $BUCKET_NAME"
|
||||
]
|
||||
"! gcloud storage buckets create --location=$REGION $BUCKET_NAME" ]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
@@ -441,8 +440,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! gsutil ls -al $BUCKET_NAME"
|
||||
]
|
||||
"! gcloud storage ls --all-versions --long $BUCKET_NAME" ]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
@@ -779,14 +777,11 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"count = ! gsutil cat $IMPORT_FILE | wc -l\n",
|
||||
"print(\"Number of Examples\", int(count[0]))\n",
|
||||
"count = ! gcloud storage cat $IMPORT_FILE | wc -l\n", "print(\"Number of Examples\", int(count[0]))\n",
|
||||
"\n",
|
||||
"print(\"First 10 rows\")\n",
|
||||
"! gsutil cat $IMPORT_FILE | head\n",
|
||||
"\n",
|
||||
"heading = ! gsutil cat $IMPORT_FILE | head -n1\n",
|
||||
"label_column = \"deaths\" # @param {type:\"string\"}\n",
|
||||
"! gcloud storage cat $IMPORT_FILE | head\n", "\n",
|
||||
"heading = ! gcloud storage cat $IMPORT_FILE | head -n1\n", "label_column = \"deaths\" # @param {type:\"string\"}\n",
|
||||
"time_column = \"date\" # @param {type:\"string\"}\n",
|
||||
"time_series_identifier_column = \"county\" # @param {type:\"string\"}\n",
|
||||
"print(\"Label Column Name\", label_column)\n",
|
||||
@@ -1358,8 +1353,7 @@
|
||||
" f.write(str(INSTANCE_2) + \"\\n\")\n",
|
||||
"\n",
|
||||
"print(gcs_input_uri)\n",
|
||||
"! gsutil cat $gcs_input_uri"
|
||||
]
|
||||
"! gcloud storage cat $gcs_input_uri" ]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
@@ -1627,8 +1621,7 @@
|
||||
"source": [
|
||||
"def get_latest_predictions(gcs_out_dir):\n",
|
||||
" \"\"\" Get the latest prediction subfolder using the timestamp in the subfolder name\"\"\"\n",
|
||||
" folders = !gsutil ls $gcs_out_dir\n",
|
||||
" latest = \"\"\n",
|
||||
" folders = !gcloud storage ls $gcs_out_dir\n", " latest = \"\"\n",
|
||||
" for folder in folders:\n",
|
||||
" subfolder = folder.split(\"/\")[-2]\n",
|
||||
" if subfolder.startswith(\"prediction-\"):\n",
|
||||
@@ -1645,10 +1638,8 @@
|
||||
" raise Exception(\"Batch Job Failed\")\n",
|
||||
" else:\n",
|
||||
" folder = get_latest_predictions(predictions)\n",
|
||||
" ! gsutil ls $folder/prediction*.csv\n",
|
||||
"\n",
|
||||
" ! gsutil cat $folder/prediction*.csv\n",
|
||||
" break\n",
|
||||
" ! gcloud storage ls $folder/prediction*.csv\n", "\n",
|
||||
" ! gcloud storage cat $folder/prediction*.csv\n", " break\n",
|
||||
" time.sleep(60)"
|
||||
]
|
||||
},
|
||||
@@ -1742,8 +1733,7 @@
|
||||
" print(e)\n",
|
||||
"\n",
|
||||
"if delete_bucket and \"BUCKET_NAME\" in globals():\n",
|
||||
" ! gsutil rm -r $BUCKET_NAME"
|
||||
]
|
||||
" ! gcloud storage rm --recursive $BUCKET_NAME" ]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
|
||||
@@ -39,7 +39,7 @@
|
||||
" </td>\n",
|
||||
" <td>\n",
|
||||
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/gapic/automl/showcase_automl_tabular_regression_online_bq.ipynb\">\n",
|
||||
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
|
||||
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
|
||||
" View on GitHub\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
@@ -397,7 +397,6 @@
|
||||
"\n",
|
||||
"from google.cloud.aiplatform import gapic as aip\n",
|
||||
"from google.protobuf import json_format\n",
|
||||
"from google.protobuf.json_format import MessageToJson, ParseDict\n",
|
||||
"from google.protobuf.struct_pb2 import Struct, Value"
|
||||
]
|
||||
},
|
||||
@@ -1614,7 +1613,7 @@
|
||||
" print(e)\n",
|
||||
"\n",
|
||||
"if delete_bucket and \"BUCKET_NAME\" in globals():\n",
|
||||
" ! gsutil rm -r $BUCKET_NAME"
|
||||
" ! gcloud storage rm --recursive $BUCKET_NAME"
|
||||
]
|
||||
}
|
||||
],
|
||||
|
||||
@@ -39,7 +39,7 @@
|
||||
" </td>\n",
|
||||
" <td>\n",
|
||||
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/gapic/automl/showcase_automl_text_classification_batch.ipynb\">\n",
|
||||
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
|
||||
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
|
||||
" View on GitHub\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
@@ -421,8 +421,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! gsutil mb -l $REGION $BUCKET_NAME"
|
||||
]
|
||||
"! gcloud storage buckets create --location $REGION $BUCKET_NAME" ]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
@@ -441,8 +440,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! gsutil ls -al $BUCKET_NAME"
|
||||
]
|
||||
"! gcloud storage ls --all-versions --long $BUCKET_NAME" ]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
@@ -881,12 +879,10 @@
|
||||
"else:\n",
|
||||
" FILE = IMPORT_FILE\n",
|
||||
"\n",
|
||||
"count = ! gsutil cat $FILE | wc -l\n",
|
||||
"print(\"Number of Examples\", int(count[0]))\n",
|
||||
"count = ! gcloud storage cat $FILE | wc -l\n", "print(\"Number of Examples\", int(count[0]))\n",
|
||||
"\n",
|
||||
"print(\"First 10 rows\")\n",
|
||||
"! gsutil cat $FILE | head"
|
||||
]
|
||||
"! gcloud storage cat $FILE | head" ]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
@@ -1314,8 +1310,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"test_items = ! gsutil cat $IMPORT_FILE | head -n2\n",
|
||||
"if len(test_items[0]) == 3:\n",
|
||||
"test_items = ! gcloud storage cat $IMPORT_FILE | head -n2\n", "if len(test_items[0]) == 3:\n",
|
||||
" _, test_item_1, test_label_1 = str(test_items[0]).split(\",\")\n",
|
||||
" _, test_item_2, test_label_2 = str(test_items[1]).split(\",\")\n",
|
||||
"else:\n",
|
||||
@@ -1371,8 +1366,7 @@
|
||||
" f.write(json.dumps(data) + \"\\n\")\n",
|
||||
"\n",
|
||||
"print(gcs_input_uri)\n",
|
||||
"! gsutil cat $gcs_input_uri"
|
||||
]
|
||||
"! gcloud storage cat $gcs_input_uri" ]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
@@ -1644,8 +1638,7 @@
|
||||
"source": [
|
||||
"def get_latest_predictions(gcs_out_dir):\n",
|
||||
" \"\"\" Get the latest prediction subfolder using the timestamp in the subfolder name\"\"\"\n",
|
||||
" folders = !gsutil ls $gcs_out_dir\n",
|
||||
" latest = \"\"\n",
|
||||
" folders = !gcloud storage ls $gcs_out_dir\n", " latest = \"\"\n",
|
||||
" for folder in folders:\n",
|
||||
" subfolder = folder.split(\"/\")[-2]\n",
|
||||
" if subfolder.startswith(\"prediction-\"):\n",
|
||||
@@ -1662,10 +1655,8 @@
|
||||
" raise Exception(\"Batch Job Failed\")\n",
|
||||
" else:\n",
|
||||
" folder = get_latest_predictions(predictions)\n",
|
||||
" ! gsutil ls $folder/prediction*.jsonl\n",
|
||||
"\n",
|
||||
" ! gsutil cat $folder/prediction*.jsonl\n",
|
||||
" break\n",
|
||||
" ! gcloud storage ls $folder/prediction*.jsonl\n", "\n",
|
||||
! gcloud storage cat $folder/prediction*.jsonl\n", " break\n",
|
||||
" time.sleep(60)"
|
||||
]
|
||||
},
|
||||
@@ -1759,8 +1750,7 @@
|
||||
" print(e)\n",
|
||||
"\n",
|
||||
"if delete_bucket and \"BUCKET_NAME\" in globals():\n",
|
||||
" ! gsutil rm -r $BUCKET_NAME"
|
||||
]
|
||||
" ! gcloud storage rm --recursive $BUCKET_NAME" ]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
|
||||
@@ -39,7 +39,7 @@
|
||||
" </td>\n",
|
||||
" <td>\n",
|
||||
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/gapic/automl/showcase_automl_text_classification_online.ipynb\">\n",
|
||||
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
|
||||
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
|
||||
" View on GitHub\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
@@ -716,12 +716,10 @@
|
||||
"else:\n",
|
||||
" FILE = IMPORT_FILE\n",
|
||||
"\n",
|
||||
"count = ! gsutil cat $FILE | wc -l\n",
|
||||
"print(\"Number of Examples\", int(count[0]))\n",
|
||||
"count = ! gcloud storage cat $FILE | wc -l\n", "print(\"Number of Examples\", int(count[0]))\n",
|
||||
"\n",
|
||||
"print(\"First 10 rows\")\n",
|
||||
"! gsutil cat $FILE | head"
|
||||
]
|
||||
"! gcloud storage cat $FILE | head" ]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
@@ -1327,7 +1325,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"test_item = ! gsutil cat $IMPORT_FILE | head -n1\n",
|
||||
"test_item = ! gcloud storage cat $IMPORT_FILE | head -n1\n",
|
||||
"if len(test_item[0]) == 3:\n",
|
||||
" _, test_item, test_label = str(test_item[0]).split(\",\")\n",
|
||||
"else:\n",
|
||||
@@ -1533,8 +1531,7 @@
|
||||
" print(e)\n",
|
||||
"\n",
|
||||
"if delete_bucket and \"BUCKET_NAME\" in globals():\n",
|
||||
" ! gsutil rm -r $BUCKET_NAME"
|
||||
]
|
||||
" ! gcloud storage rm --recursive $BUCKET_NAME" ]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
|
||||
+12
-14
@@ -39,7 +39,7 @@
|
||||
" </td>\n",
|
||||
" <td>\n",
|
||||
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/gapic/automl/showcase_automl_text_entity_extraction_batch.ipynb\">\n",
|
||||
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
|
||||
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
|
||||
" View on GitHub\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
@@ -421,7 +421,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! gsutil mb -l $REGION $BUCKET_NAME"
|
||||
"! gcloud storage buckets create --location $REGION $BUCKET_NAME"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -441,7 +441,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! gsutil ls -al $BUCKET_NAME"
|
||||
"! gcloud storage ls --all-versions --long $BUCKET_NAME"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -478,9 +478,7 @@
|
||||
"import time\n",
|
||||
"\n",
|
||||
"from google.cloud.aiplatform import gapic as aip\n",
|
||||
"from google.protobuf import json_format\n",
|
||||
"from google.protobuf.json_format import MessageToJson, ParseDict\n",
|
||||
"from google.protobuf.struct_pb2 import Struct, Value"
|
||||
"from google.protobuf import json_format"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -888,11 +886,11 @@
|
||||
"else:\n",
|
||||
" FILE = IMPORT_FILE\n",
|
||||
"\n",
|
||||
"count = ! gsutil cat $FILE | wc -l\n",
|
||||
"count = ! gcloud storage cat $FILE | wc -l\n",
|
||||
"print(\"Number of Examples\", int(count[0]))\n",
|
||||
"\n",
|
||||
"print(\"First 10 rows\")\n",
|
||||
"! gsutil cat $FILE | head"
|
||||
"! gcloud storage cat $FILE | head"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -1377,7 +1375,7 @@
|
||||
" f.write(json.dumps(data) + \"\\n\")\n",
|
||||
"\n",
|
||||
"print(gcs_input_uri)\n",
|
||||
"! gsutil cat $gcs_input_uri"
|
||||
"! gcloud storage cat $gcs_input_uri"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -1652,8 +1650,8 @@
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def get_latest_predictions(gcs_out_dir):\n",
|
||||
" \"\"\" Get the latest prediction subfolder using the timestamp in the subfolder name\"\"\"\n",
|
||||
" folders = !gsutil ls $gcs_out_dir\n",
|
||||
" \"\"\"Get the latest prediction subfolder using the timestamp in the subfolder name\"\"\"\n",
|
||||
" folders = !gcloud storage ls $gcs_out_dir\n",
|
||||
" latest = \"\"\n",
|
||||
" for folder in folders:\n",
|
||||
" subfolder = folder.split(\"/\")[-2]\n",
|
||||
@@ -1671,9 +1669,9 @@
|
||||
" raise Exception(\"Batch Job Failed\")\n",
|
||||
" else:\n",
|
||||
" folder = get_latest_predictions(predictions)\n",
|
||||
" ! gsutil ls $folder/prediction*.jsonl\n",
|
||||
" ! gcloud storage ls $folder/prediction*.jsonl\n",
|
||||
"\n",
|
||||
" ! gsutil cat $folder/prediction*.jsonl\n",
|
||||
" ! gcloud storage cat $folder/prediction*.jsonl\n",
|
||||
" break\n",
|
||||
" time.sleep(60)"
|
||||
]
|
||||
@@ -1768,7 +1766,7 @@
|
||||
" print(e)\n",
|
||||
"\n",
|
||||
"if delete_bucket and \"BUCKET_NAME\" in globals():\n",
|
||||
" ! gsutil rm -r $BUCKET_NAME"
|
||||
" ! gcloud storage rm --recursive $BUCKET_NAME"
|
||||
]
|
||||
}
|
||||
],
|
||||
|
||||
+4
-7
@@ -39,7 +39,7 @@
|
||||
" </td>\n",
|
||||
" <td>\n",
|
||||
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/gapic/automl/showcase_automl_text_entity_extraction_online.ipynb\">\n",
|
||||
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
|
||||
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
|
||||
" View on GitHub\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
@@ -723,12 +723,10 @@
|
||||
"else:\n",
|
||||
" FILE = IMPORT_FILE\n",
|
||||
"\n",
|
||||
"count = ! gsutil cat $FILE | wc -l\n",
|
||||
"print(\"Number of Examples\", int(count[0]))\n",
|
||||
"count = ! gcloud storage cat $FILE | wc -l\n", "print(\"Number of Examples\", int(count[0]))\n",
|
||||
"\n",
|
||||
"print(\"First 10 rows\")\n",
|
||||
"! gsutil cat $FILE | head"
|
||||
]
|
||||
"! gcloud storage cat $FILE | head" ]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
@@ -1544,8 +1542,7 @@
|
||||
" print(e)\n",
|
||||
"\n",
|
||||
"if delete_bucket and \"BUCKET_NAME\" in globals():\n",
|
||||
" ! gsutil rm -r $BUCKET_NAME"
|
||||
]
|
||||
" ! gcloud storage rm --recursive $BUCKET_NAME" ]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
|
||||
+11
-21
@@ -39,7 +39,7 @@
|
||||
" </td>\n",
|
||||
" <td>\n",
|
||||
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/gapic/automl/showcase_automl_text_multi-label_classification_batch.ipynb\">\n",
|
||||
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
|
||||
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
|
||||
" View on GitHub\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
@@ -421,8 +421,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! gsutil mb -l $REGION $BUCKET_NAME"
|
||||
]
|
||||
"! gcloud storage buckets create --location=$REGION $BUCKET_NAME" ]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
@@ -441,8 +440,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! gsutil ls -al $BUCKET_NAME"
|
||||
]
|
||||
"! gcloud storage ls --all-versions --long $BUCKET_NAME" ]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
@@ -881,12 +879,10 @@
|
||||
"else:\n",
|
||||
" FILE = IMPORT_FILE\n",
|
||||
"\n",
|
||||
"count = ! gsutil cat $FILE | wc -l\n",
|
||||
"print(\"Number of Examples\", int(count[0]))\n",
|
||||
"count = ! gcloud storage cat $FILE | wc -l\n", "print(\"Number of Examples\", int(count[0]))\n",
|
||||
"\n",
|
||||
"print(\"First 10 rows\")\n",
|
||||
"! gsutil cat $FILE | head"
|
||||
]
|
||||
"! gcloud storage cat $FILE | head" ]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
@@ -1314,8 +1310,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"test_items = ! gsutil cat $IMPORT_FILE | head -n2\n",
|
||||
"\n",
|
||||
"test_items = ! gcloud storage cat $IMPORT_FILE | head -n2\n", "\n",
|
||||
"cols_1 = str(test_items[0]).split(\",\")\n",
|
||||
"cols_2 = str(test_items[1]).split(\",\")\n",
|
||||
"test_item_1 = cols_1[0]\n",
|
||||
@@ -1372,8 +1367,7 @@
|
||||
" f.write(json.dumps(data) + \"\\n\")\n",
|
||||
"\n",
|
||||
"print(gcs_input_uri)\n",
|
||||
"! gsutil cat $gcs_input_uri"
|
||||
]
|
||||
"! gcloud storage cat $gcs_input_uri" ]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
@@ -1645,8 +1639,7 @@
|
||||
"source": [
|
||||
"def get_latest_predictions(gcs_out_dir):\n",
|
||||
" \"\"\" Get the latest prediction subfolder using the timestamp in the subfolder name\"\"\"\n",
|
||||
" folders = !gsutil ls $gcs_out_dir\n",
|
||||
" latest = \"\"\n",
|
||||
" folders = !gcloud storage ls $gcs_out_dir\n", " latest = \"\"\n",
|
||||
" for folder in folders:\n",
|
||||
" subfolder = folder.split(\"/\")[-2]\n",
|
||||
" if subfolder.startswith(\"prediction-\"):\n",
|
||||
@@ -1663,10 +1656,8 @@
|
||||
" raise Exception(\"Batch Job Failed\")\n",
|
||||
" else:\n",
|
||||
" folder = get_latest_predictions(predictions)\n",
|
||||
" ! gsutil ls $folder/prediction*.jsonl\n",
|
||||
"\n",
|
||||
" ! gsutil cat $folder/prediction*.jsonl\n",
|
||||
" break\n",
|
||||
" ! gcloud storage ls $folder/prediction*.jsonl\n", "\n",
|
||||
" ! gcloud storage cat $folder/prediction*.jsonl\n", " break\n",
|
||||
" time.sleep(60)"
|
||||
]
|
||||
},
|
||||
@@ -1760,8 +1751,7 @@
|
||||
" print(e)\n",
|
||||
"\n",
|
||||
"if delete_bucket and \"BUCKET_NAME\" in globals():\n",
|
||||
" ! gsutil rm -r $BUCKET_NAME"
|
||||
]
|
||||
" ! gcloud storage rm --recursive $BUCKET_NAME" ]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
|
||||
+5
-9
@@ -39,7 +39,7 @@
|
||||
" </td>\n",
|
||||
" <td>\n",
|
||||
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/gapic/automl/showcase_automl_text_multi-label_classification_online.ipynb\">\n",
|
||||
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
|
||||
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
|
||||
" View on GitHub\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
@@ -716,12 +716,10 @@
|
||||
"else:\n",
|
||||
" FILE = IMPORT_FILE\n",
|
||||
"\n",
|
||||
"count = ! gsutil cat $FILE | wc -l\n",
|
||||
"print(\"Number of Examples\", int(count[0]))\n",
|
||||
"count = ! gcloud storage cat $FILE | wc -l\n", "print(\"Number of Examples\", int(count[0]))\n",
|
||||
"\n",
|
||||
"print(\"First 10 rows\")\n",
|
||||
"! gsutil cat $FILE | head"
|
||||
]
|
||||
"! gcloud storage cat $FILE | head" ]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
@@ -1327,8 +1325,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"test_item = ! gsutil cat $IMPORT_FILE | head -n1\n",
|
||||
"\n",
|
||||
"test_item = ! gcloud storage cat $IMPORT_FILE | head -n1\n", "\n",
|
||||
"cols = str(test_item[0]).split(\",\")\n",
|
||||
"test_item = cols[0]\n",
|
||||
"test_label = cols[1:]\n",
|
||||
@@ -1533,8 +1530,7 @@
|
||||
" print(e)\n",
|
||||
"\n",
|
||||
"if delete_bucket and \"BUCKET_NAME\" in globals():\n",
|
||||
" ! gsutil rm -r $BUCKET_NAME"
|
||||
]
|
||||
" ! gcloud storage rm --recursive $BUCKET_NAME" ]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
|
||||
+11
-21
@@ -39,7 +39,7 @@
|
||||
" </td>\n",
|
||||
" <td>\n",
|
||||
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/gapic/automl/showcase_automl_text_sentiment_analysis_batch.ipynb\">\n",
|
||||
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
|
||||
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
|
||||
" View on GitHub\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
@@ -421,8 +421,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! gsutil mb -l $REGION $BUCKET_NAME"
|
||||
]
|
||||
"! gcloud storage buckets create --location $REGION $BUCKET_NAME" ]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
@@ -441,8 +440,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! gsutil ls -al $BUCKET_NAME"
|
||||
]
|
||||
"! gcloud storage ls --all-versions --long $BUCKET_NAME" ]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
@@ -883,12 +881,10 @@
|
||||
"else:\n",
|
||||
" FILE = IMPORT_FILE\n",
|
||||
"\n",
|
||||
"count = ! gsutil cat $FILE | wc -l\n",
|
||||
"print(\"Number of Examples\", int(count[0]))\n",
|
||||
"count = ! gcloud storage cat $FILE | wc -l\n", "print(\"Number of Examples\", int(count[0]))\n",
|
||||
"\n",
|
||||
"print(\"First 10 rows\")\n",
|
||||
"! gsutil cat $FILE | head"
|
||||
]
|
||||
"! gcloud storage cat $FILE | head" ]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
@@ -1316,8 +1312,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"test_items = ! gsutil cat $IMPORT_FILE | head -n2\n",
|
||||
"\n",
|
||||
"test_items = ! gcloud storage cat $IMPORT_FILE | head -n2\n", "\n",
|
||||
"if len(test_items[0]) == 4:\n",
|
||||
" _, test_item_1, test_label_1, _ = str(test_items[0]).split(\",\")\n",
|
||||
" _, test_item_2, test_label_2, _ = str(test_items[1]).split(\",\")\n",
|
||||
@@ -1375,8 +1370,7 @@
|
||||
" f.write(json.dumps(data) + \"\\n\")\n",
|
||||
"\n",
|
||||
"print(gcs_input_uri)\n",
|
||||
"! gsutil cat $gcs_input_uri"
|
||||
]
|
||||
"! gcloud storage cat $gcs_input_uri" ]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
@@ -1643,8 +1637,7 @@
|
||||
"source": [
|
||||
"def get_latest_predictions(gcs_out_dir):\n",
|
||||
" \"\"\" Get the latest prediction subfolder using the timestamp in the subfolder name\"\"\"\n",
|
||||
" folders = !gsutil ls $gcs_out_dir\n",
|
||||
" latest = \"\"\n",
|
||||
" folders = !gcloud storage ls $gcs_out_dir\n", " latest = \"\"\n",
|
||||
" for folder in folders:\n",
|
||||
" subfolder = folder.split(\"/\")[-2]\n",
|
||||
" if subfolder.startswith(\"prediction-\"):\n",
|
||||
@@ -1661,10 +1654,8 @@
|
||||
" raise Exception(\"Batch Job Failed\")\n",
|
||||
" else:\n",
|
||||
" folder = get_latest_predictions(predictions)\n",
|
||||
" ! gsutil ls $folder/prediction*.jsonl\n",
|
||||
"\n",
|
||||
" ! gsutil cat $folder/prediction*.jsonl\n",
|
||||
" break\n",
|
||||
" ! gcloud storage ls $folder/prediction*.jsonl\n", "\n",
|
||||
" ! gcloud storage cat $folder/prediction*.jsonl\n", " break\n",
|
||||
" time.sleep(60)"
|
||||
]
|
||||
},
|
||||
@@ -1758,8 +1749,7 @@
|
||||
" print(e)\n",
|
||||
"\n",
|
||||
"if delete_bucket and \"BUCKET_NAME\" in globals():\n",
|
||||
" ! gsutil rm -r $BUCKET_NAME"
|
||||
]
|
||||
" ! gcloud storage rm --recursive $BUCKET_NAME" ]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
|
||||
+5
-9
@@ -39,7 +39,7 @@
|
||||
" </td>\n",
|
||||
" <td>\n",
|
||||
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/gapic/automl/showcase_automl_text_sentiment_analysis_online.ipynb\">\n",
|
||||
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
|
||||
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
|
||||
" View on GitHub\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
@@ -718,12 +718,10 @@
|
||||
"else:\n",
|
||||
" FILE = IMPORT_FILE\n",
|
||||
"\n",
|
||||
"count = ! gsutil cat $FILE | wc -l\n",
|
||||
"print(\"Number of Examples\", int(count[0]))\n",
|
||||
"count = ! gcloud storage cat $FILE | wc -l\n", "print(\"Number of Examples\", int(count[0]))\n",
|
||||
"\n",
|
||||
"print(\"First 10 rows\")\n",
|
||||
"! gsutil cat $FILE | head"
|
||||
]
|
||||
"! gcloud storage cat $FILE | head" ]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
@@ -1329,8 +1327,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"test_item = ! gsutil cat $IMPORT_FILE | head -n1\n",
|
||||
"if len(test_item[0]) == 3:\n",
|
||||
"test_item = ! gcloud storage cat $IMPORT_FILE | head -n1\n", "if len(test_item[0]) == 3:\n",
|
||||
" _, test_item, test_label, max = str(test_item[0]).split(\",\")\n",
|
||||
"else:\n",
|
||||
" test_item, test_label, max = str(test_item[0]).split(\",\")\n",
|
||||
@@ -1534,8 +1531,7 @@
|
||||
" print(e)\n",
|
||||
"\n",
|
||||
"if delete_bucket and \"BUCKET_NAME\" in globals():\n",
|
||||
" ! gsutil rm -r $BUCKET_NAME"
|
||||
]
|
||||
" ! gcloud storage rm --recursive $BUCKET_NAME" ]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
|
||||
+11
-21
@@ -39,7 +39,7 @@
|
||||
" </td>\n",
|
||||
" <td>\n",
|
||||
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/gapic/automl/showcase_automl_video_action_recognition_batch.ipynb\">\n",
|
||||
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
|
||||
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
|
||||
" View on GitHub\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
@@ -420,8 +420,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! gsutil mb -l $REGION $BUCKET_NAME"
|
||||
]
|
||||
"! gcloud storage buckets create --location=$REGION $BUCKET_NAME" ]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
@@ -440,8 +439,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! gsutil ls -al $BUCKET_NAME"
|
||||
]
|
||||
"! gcloud storage ls --all-versions --long $BUCKET_NAME" ]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
@@ -888,12 +886,10 @@
|
||||
"else:\n",
|
||||
" FILE = IMPORT_FILE\n",
|
||||
"\n",
|
||||
"count = ! gsutil cat $FILE | wc -l\n",
|
||||
"print(\"Number of Examples\", int(count[0]))\n",
|
||||
"count = ! gcloud storage cat $FILE | wc -l\n", "print(\"Number of Examples\", int(count[0]))\n",
|
||||
"\n",
|
||||
"print(\"First 10 rows\")\n",
|
||||
"! gsutil cat $FILE | head"
|
||||
]
|
||||
"! gcloud storage cat $FILE | head" ]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
@@ -1310,8 +1306,7 @@
|
||||
"import json\n",
|
||||
"\n",
|
||||
"import_file = IMPORT_FILES[0]\n",
|
||||
"test_items = ! gsutil cat $import_file | head -n2\n",
|
||||
"\n",
|
||||
"test_items = ! gcloud storage cat $import_file | head -n2\n", "\n",
|
||||
"cols = str(test_items[0]).split(',')\n",
|
||||
"test_item_1 = str(cols[0])\n",
|
||||
"test_label_1 = str(cols[-1])\n",
|
||||
@@ -1360,8 +1355,7 @@
|
||||
" f.write(json.dumps(data) + '\\n')\n",
|
||||
"\n",
|
||||
"print(gcs_input_uri)\n",
|
||||
"! gsutil cat $gcs_input_uri"
|
||||
]
|
||||
"! gcloud storage cat $gcs_input_uri" ]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
@@ -1637,8 +1631,7 @@
|
||||
"source": [
|
||||
"def get_latest_predictions(gcs_out_dir):\n",
|
||||
" ''' Get the latest prediction subfolder using the timestamp in the subfolder name'''\n",
|
||||
" folders = !gsutil ls $gcs_out_dir\n",
|
||||
" latest = \"\"\n",
|
||||
" folders = !gcloud storage ls $gcs_out_dir\n", " latest = \"\"\n",
|
||||
" for folder in folders:\n",
|
||||
" subfolder = folder.split('/')[-2]\n",
|
||||
" if subfolder.startswith('prediction-'):\n",
|
||||
@@ -1655,10 +1648,8 @@
|
||||
" raise Exception(\"Batch Job Failed\")\n",
|
||||
" else:\n",
|
||||
" folder = get_latest_predictions(predictions)\n",
|
||||
" ! gsutil ls $folder/prediction*.jsonl\n",
|
||||
"\n",
|
||||
" ! gsutil cat $folder/prediction*.jsonl\n",
|
||||
" break\n",
|
||||
" ! gcloud storage ls $folder/prediction*.jsonl\n", "\n",
|
||||
" ! gcloud storage cat $folder/prediction*.jsonl\n", " break\n",
|
||||
" time.sleep(60)"
|
||||
]
|
||||
},
|
||||
@@ -1752,8 +1743,7 @@
|
||||
" print(e)\n",
|
||||
"\n",
|
||||
"if delete_bucket and 'BUCKET_NAME' in globals():\n",
|
||||
" ! gsutil rm -r $BUCKET_NAME"
|
||||
]
|
||||
" ! gcloud storage rm --recursive $BUCKET_NAME" ]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
|
||||
@@ -39,7 +39,7 @@
|
||||
" </td>\n",
|
||||
" <td>\n",
|
||||
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/gapic/automl/showcase_automl_video_classification_batch.ipynb\">\n",
|
||||
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
|
||||
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
|
||||
" View on GitHub\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
@@ -420,8 +420,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! gsutil mb -l $REGION $BUCKET_NAME"
|
||||
]
|
||||
"! gcloud storage buckets create --location $REGION $BUCKET_NAME" ]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
@@ -440,8 +439,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! gsutil ls -al $BUCKET_NAME"
|
||||
]
|
||||
"! gcloud storage ls --all-versions --long $BUCKET_NAME" ]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
@@ -885,12 +883,10 @@
|
||||
"else:\n",
|
||||
" FILE = IMPORT_FILE\n",
|
||||
"\n",
|
||||
"count = ! gsutil cat $FILE | wc -l\n",
|
||||
"print(\"Number of Examples\", int(count[0]))\n",
|
||||
"count = ! gcloud storage cat $FILE | wc -l\n", "print(\"Number of Examples\", int(count[0]))\n",
|
||||
"\n",
|
||||
"print(\"First 10 rows\")\n",
|
||||
"! gsutil cat $FILE | head"
|
||||
]
|
||||
"! gcloud storage cat $FILE | head" ]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
@@ -1301,8 +1297,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"test_items = ! gsutil cat $IMPORT_FILE | head -n2\n",
|
||||
"\n",
|
||||
"test_items = ! gcloud storage cat $IMPORT_FILE | head -n2\n", "\n",
|
||||
"if len(test_items[0]) == 5:\n",
|
||||
" _, test_item_1, test_label_1, _, _ = str(test_items[0]).split(',')\n",
|
||||
" _, test_item_2, test_label_2, _, _ = str(test_items[1]).split(',')\n",
|
||||
@@ -1351,8 +1346,7 @@
|
||||
" f.write(json.dumps(data) + '\\n')\n",
|
||||
"\n",
|
||||
"print(gcs_input_uri)\n",
|
||||
"! gsutil cat $gcs_input_uri"
|
||||
]
|
||||
"! gcloud storage cat $gcs_input_uri" ]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
@@ -1630,8 +1624,7 @@
|
||||
"source": [
|
||||
"def get_latest_predictions(gcs_out_dir):\n",
|
||||
" ''' Get the latest prediction subfolder using the timestamp in the subfolder name'''\n",
|
||||
" folders = !gsutil ls $gcs_out_dir\n",
|
||||
" latest = \"\"\n",
|
||||
" folders = !gcloud storage ls $gcs_out_dir\n", " latest = \"\"\n",
|
||||
" for folder in folders:\n",
|
||||
" subfolder = folder.split('/')[-2]\n",
|
||||
" if subfolder.startswith('prediction-'):\n",
|
||||
@@ -1648,10 +1641,8 @@
|
||||
" raise Exception(\"Batch Job Failed\")\n",
|
||||
" else:\n",
|
||||
" folder = get_latest_predictions(predictions)\n",
|
||||
" ! gsutil ls $folder/prediction*.jsonl\n",
|
||||
"\n",
|
||||
" ! gsutil cat $folder/prediction*.jsonl\n",
|
||||
" break\n",
|
||||
" ! gcloud storage ls $folder/prediction*.jsonl\n", "\n",
|
||||
"! gcloud storage cat $folder/prediction*.jsonl\n", " break\n",
|
||||
" time.sleep(60)"
|
||||
]
|
||||
},
|
||||
@@ -1745,8 +1736,7 @@
|
||||
" print(e)\n",
|
||||
"\n",
|
||||
"if delete_bucket and 'BUCKET_NAME' in globals():\n",
|
||||
" ! gsutil rm -r $BUCKET_NAME"
|
||||
]
|
||||
" ! gcloud storage rm --recursive $BUCKET_NAME" ]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
|
||||
+11
-21
@@ -39,7 +39,7 @@
|
||||
" </td>\n",
|
||||
" <td>\n",
|
||||
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/gapic/automl/showcase_automl_video_object_tracking_batch.ipynb\">\n",
|
||||
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
|
||||
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
|
||||
" View on GitHub\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
@@ -420,8 +420,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! gsutil mb -l $REGION $BUCKET_NAME"
|
||||
]
|
||||
"! gcloud storage buckets create --location $REGION $BUCKET_NAME" ]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
@@ -440,8 +439,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! gsutil ls -al $BUCKET_NAME"
|
||||
]
|
||||
"! gcloud storage ls --all-versions --long $BUCKET_NAME" ]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
@@ -890,12 +888,10 @@
|
||||
"else:\n",
|
||||
" FILE = IMPORT_FILE\n",
|
||||
"\n",
|
||||
"count = ! gsutil cat $FILE | wc -l\n",
|
||||
"print(\"Number of Examples\", int(count[0]))\n",
|
||||
"count = ! gcloud storage cat $FILE | wc -l\n", "print(\"Number of Examples\", int(count[0]))\n",
|
||||
"\n",
|
||||
"print(\"First 10 rows\")\n",
|
||||
"! gsutil cat $FILE | head"
|
||||
]
|
||||
"! gcloud storage cat $FILE | head" ]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
@@ -1309,8 +1305,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"test_items = ! gsutil cat $IMPORT_FILE | head -n2\n",
|
||||
"\n",
|
||||
"test_items = ! gcloud storage cat $IMPORT_FILE | head -n2\n", "\n",
|
||||
"cols_1 = test_items[0].split(',')\n",
|
||||
"cols_2 = test_items[1].split(',')\n",
|
||||
"if len(cols_1) > 12:\n",
|
||||
@@ -1365,8 +1360,7 @@
|
||||
" f.write(json.dumps(data) + '\\n')\n",
|
||||
"\n",
|
||||
"print(gcs_input_uri)\n",
|
||||
"! gsutil cat $gcs_input_uri"
|
||||
]
|
||||
"! gcloud storage cat $gcs_input_uri" ]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
@@ -1638,8 +1632,7 @@
|
||||
"source": [
|
||||
"def get_latest_predictions(gcs_out_dir):\n",
|
||||
" ''' Get the latest prediction subfolder using the timestamp in the subfolder name'''\n",
|
||||
" folders = !gsutil ls $gcs_out_dir\n",
|
||||
" latest = \"\"\n",
|
||||
" folders = !gcloud storage ls $gcs_out_dir\n", " latest = \"\"\n",
|
||||
" for folder in folders:\n",
|
||||
" subfolder = folder.split('/')[-2]\n",
|
||||
" if subfolder.startswith('prediction-'):\n",
|
||||
@@ -1656,10 +1649,8 @@
|
||||
" raise Exception(\"Batch Job Failed\")\n",
|
||||
" else:\n",
|
||||
" folder = get_latest_predictions(predictions)\n",
|
||||
" ! gsutil ls $folder/prediction*.jsonl\n",
|
||||
"\n",
|
||||
" ! gsutil cat $folder/prediction*.jsonl\n",
|
||||
" break\n",
|
||||
" ! gcloud storage ls $folder/prediction*.jsonl\n", "\n",
|
||||
" ! gcloud storage cat $folder/prediction*.jsonl\n", " break\n",
|
||||
" time.sleep(60)"
|
||||
]
|
||||
},
|
||||
@@ -1753,8 +1744,7 @@
|
||||
" print(e)\n",
|
||||
"\n",
|
||||
"if delete_bucket and 'BUCKET_NAME' in globals():\n",
|
||||
" ! gsutil rm -r $BUCKET_NAME"
|
||||
]
|
||||
" ! gcloud storage rm --recursive $BUCKET_NAME" ]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
|
||||
@@ -39,7 +39,7 @@
|
||||
" </td>\n",
|
||||
" <td>\n",
|
||||
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/gapic/custom/showcase_custom_image_classification_batch.ipynb\">\n",
|
||||
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
|
||||
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
|
||||
" View on GitHub\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
|
||||
+1
-1
@@ -39,7 +39,7 @@
|
||||
" </td>\n",
|
||||
" <td>\n",
|
||||
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/gapic/custom/showcase_custom_image_classification_batch_explain.ipynb\">\n",
|
||||
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
|
||||
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
|
||||
" View on GitHub\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
|
||||
@@ -39,7 +39,7 @@
|
||||
" </td>\n",
|
||||
" <td>\n",
|
||||
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/gapic/custom/showcase_custom_image_classification_online.ipynb\">\n",
|
||||
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
|
||||
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
|
||||
" View on GitHub\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
|
||||
+1
-1
@@ -39,7 +39,7 @@
|
||||
" </td>\n",
|
||||
" <td>\n",
|
||||
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/gapic/custom/showcase_custom_image_classification_online_ab_testing.ipynb\">\n",
|
||||
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
|
||||
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
|
||||
" View on GitHub\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
|
||||
+1
-1
@@ -39,7 +39,7 @@
|
||||
" </td>\n",
|
||||
" <td>\n",
|
||||
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/gapic/custom/showcase_custom_image_classification_online_container.ipynb\">\n",
|
||||
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
|
||||
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
|
||||
" View on GitHub\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
|
||||
+1
-1
@@ -39,7 +39,7 @@
|
||||
" </td>\n",
|
||||
" <td>\n",
|
||||
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/gapic/custom/showcase_custom_image_classification_online_explain.ipynb\">\n",
|
||||
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
|
||||
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
|
||||
" View on GitHub\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
|
||||
+1
-1
@@ -40,7 +40,7 @@
|
||||
" </td>\n",
|
||||
" <td>\n",
|
||||
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/gapic/custom/showcase_custom_image_classification_online_explain_example_based_api.ipynb\">\n",
|
||||
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
|
||||
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
|
||||
" View on GitHub\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
|
||||
+1
-1
@@ -39,7 +39,7 @@
|
||||
" </td>\n",
|
||||
" <td>\n",
|
||||
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/gapic/custom/showcase_custom_image_classification_online_exported_ds.ipynb\">\n",
|
||||
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
|
||||
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
|
||||
" View on GitHub\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
|
||||
+1
-1
@@ -39,7 +39,7 @@
|
||||
" </td>\n",
|
||||
" <td>\n",
|
||||
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/gapic/custom/showcase_custom_image_classification_online_pipeline.ipynb\">\n",
|
||||
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
|
||||
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
|
||||
" View on GitHub\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
|
||||
+1
-1
@@ -39,7 +39,7 @@
|
||||
" </td>\n",
|
||||
" <td>\n",
|
||||
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/gapic/custom/showcase_custom_image_classification_online_raw.ipynb\">\n",
|
||||
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
|
||||
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
|
||||
" View on GitHub\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
|
||||
+1
-1
@@ -39,7 +39,7 @@
|
||||
" </td>\n",
|
||||
" <td>\n",
|
||||
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/gapic/custom/showcase_custom_image_classification_online_tfserving.ipynb\">\n",
|
||||
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
|
||||
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
|
||||
" View on GitHub\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
|
||||
+1
-1
@@ -39,7 +39,7 @@
|
||||
" </td>\n",
|
||||
" <td>\n",
|
||||
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/gapic/custom/showcase_custom_image_super_resolution_online_post.ipynb\">\n",
|
||||
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
|
||||
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
|
||||
" View on GitHub\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
|
||||
+1
-1
@@ -39,7 +39,7 @@
|
||||
" </td>\n",
|
||||
" <td>\n",
|
||||
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/gapic/custom/showcase_custom_tabular_classification_online_exported_ds.ipynb\">\n",
|
||||
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
|
||||
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
|
||||
" View on GitHub\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
|
||||
@@ -39,7 +39,7 @@
|
||||
" </td>\n",
|
||||
" <td>\n",
|
||||
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/gapic/custom/showcase_custom_tabular_regression_batch.ipynb\">\n",
|
||||
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
|
||||
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
|
||||
" View on GitHub\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
|
||||
+1
-1
@@ -39,7 +39,7 @@
|
||||
" </td>\n",
|
||||
" <td>\n",
|
||||
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/gapic/custom/showcase_custom_tabular_regression_batch_explain.ipynb\">\n",
|
||||
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
|
||||
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
|
||||
" View on GitHub\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
|
||||
@@ -39,7 +39,7 @@
|
||||
" </td>\n",
|
||||
" <td>\n",
|
||||
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/gapic/custom/showcase_custom_tabular_regression_online.ipynb\">\n",
|
||||
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
|
||||
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
|
||||
" View on GitHub\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
|
||||
+1
-1
@@ -39,7 +39,7 @@
|
||||
" </td>\n",
|
||||
" <td>\n",
|
||||
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/gapic/custom/showcase_custom_tabular_regression_online_container.ipynb\">\n",
|
||||
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
|
||||
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
|
||||
" View on GitHub\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
|
||||
+1
-1
@@ -39,7 +39,7 @@
|
||||
" </td>\n",
|
||||
" <td>\n",
|
||||
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/gapic/custom/showcase_custom_tabular_regression_online_explain.ipynb\">\n",
|
||||
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
|
||||
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
|
||||
" View on GitHub\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
|
||||
+1
-1
@@ -39,7 +39,7 @@
|
||||
" </td>\n",
|
||||
" <td>\n",
|
||||
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/gapic/custom/showcase_custom_tabular_regression_online_pipeline.ipynb\">\n",
|
||||
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
|
||||
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
|
||||
" View on GitHub\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
|
||||
+1
-1
@@ -39,7 +39,7 @@
|
||||
" </td>\n",
|
||||
" <td>\n",
|
||||
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/gapic/custom/showcase_custom_tabular_regression_online_tfserving.ipynb\">\n",
|
||||
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
|
||||
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
|
||||
" View on GitHub\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
|
||||
+1
-1
@@ -39,7 +39,7 @@
|
||||
" </td>\n",
|
||||
" <td>\n",
|
||||
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/gapic/custom/showcase_custom_text_binary_classification_batch.ipynb\">\n",
|
||||
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
|
||||
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
|
||||
" View on GitHub\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
|
||||
+1
-1
@@ -39,7 +39,7 @@
|
||||
" </td>\n",
|
||||
" <td>\n",
|
||||
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/gapic/custom/showcase_custom_text_binary_classification_online.ipynb\">\n",
|
||||
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
|
||||
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
|
||||
" View on GitHub\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
|
||||
+1
-1
@@ -39,7 +39,7 @@
|
||||
" </td>\n",
|
||||
" <td>\n",
|
||||
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/gapic/custom/showcase_custom_text_binary_classification_online_container.ipynb\">\n",
|
||||
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
|
||||
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
|
||||
" View on GitHub\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
|
||||
+1
-1
@@ -39,7 +39,7 @@
|
||||
" </td>\n",
|
||||
" <td>\n",
|
||||
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/gapic/custom/showcase_custom_text_binary_classification_online_pipeline.ipynb\">\n",
|
||||
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
|
||||
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
|
||||
" View on GitHub\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
|
||||
+1
-1
@@ -39,7 +39,7 @@
|
||||
" </td>\n",
|
||||
" <td>\n",
|
||||
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/gapic/custom/showcase_custom_text_binary_classification_online_tfserving.ipynb\">\n",
|
||||
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
|
||||
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
|
||||
" View on GitHub\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
|
||||
+7
-16
@@ -39,7 +39,7 @@
|
||||
" </td>\n",
|
||||
" <td>\n",
|
||||
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/gapic/custom/showcase_custom_text_classification_online_exported_ds.ipynb\">\n",
|
||||
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
|
||||
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
|
||||
" View on GitHub\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
@@ -426,8 +426,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! gsutil mb -l $REGION $BUCKET_NAME"
|
||||
]
|
||||
"! gcloud storage buckets create --location=$REGION $BUCKET_NAME" ]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
@@ -446,8 +445,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! gsutil ls -al $BUCKET_NAME"
|
||||
]
|
||||
"! gcloud storage ls --all-versions --long $BUCKET_NAME" ]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
@@ -483,9 +481,6 @@
|
||||
"import time\n",
|
||||
"\n",
|
||||
"from google.cloud.aiplatform import gapic as aip\n",
|
||||
"from google.protobuf import json_format\n",
|
||||
"from google.protobuf.json_format import MessageToJson, ParseDict\n",
|
||||
"from google.protobuf.struct_pb2 import Struct, Value"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -991,12 +986,10 @@
|
||||
"else:\n",
|
||||
" FILE = IMPORT_FILE\n",
|
||||
"\n",
|
||||
"count = ! gsutil cat $FILE | wc -l\n",
|
||||
"print(\"Number of Examples\", int(count[0]))\n",
|
||||
"count = ! gcloud storage cat $FILE | wc -l\n", "print(\"Number of Examples\", int(count[0]))\n",
|
||||
"\n",
|
||||
"print(\"First 10 rows\")\n",
|
||||
"! gsutil cat $FILE | head"
|
||||
]
|
||||
"! gcloud storage cat $FILE | head" ]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
@@ -1153,8 +1146,7 @@
|
||||
"source": [
|
||||
"jsonl_index = result.exported_files[0]\n",
|
||||
"\n",
|
||||
"! gsutil cat $jsonl_index | head"
|
||||
]
|
||||
"! gcloud storage cat $jsonl_index | head" ]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
@@ -1286,8 +1278,7 @@
|
||||
" print(e)\n",
|
||||
"\n",
|
||||
"if delete_bucket and \"BUCKET_NAME\" in globals():\n",
|
||||
" ! gsutil rm -r $BUCKET_NAME"
|
||||
]
|
||||
" ! gcloud storage rm --recursive $BUCKET_NAME" ]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
|
||||
+6
-11
@@ -39,7 +39,7 @@
|
||||
" </td>\n",
|
||||
" <td>\n",
|
||||
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/gapic/custom/showcase_hyperparmeter_tuning_image_classification.ipynb\">\n",
|
||||
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
|
||||
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
|
||||
" View on GitHub\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
@@ -419,8 +419,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! gsutil mb -l $REGION $BUCKET_NAME"
|
||||
]
|
||||
"! gcloud storage buckets create --location=$REGION $BUCKET_NAME" ]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
@@ -439,8 +438,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! gsutil ls -al $BUCKET_NAME"
|
||||
]
|
||||
"! gcloud storage ls --all-versions --long $BUCKET_NAME" ]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
@@ -1119,8 +1117,7 @@
|
||||
"! rm -f custom.tar custom.tar.gz\n",
|
||||
"! tar cvf custom.tar custom\n",
|
||||
"! gzip custom.tar\n",
|
||||
"! gsutil cp custom.tar.gz $BUCKET_NAME/trainer_cifar10.tar.gz"
|
||||
]
|
||||
"! gcloud storage cp custom.tar.gz $BUCKET_NAME/trainer_cifar10.tar.gz" ]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
@@ -1617,8 +1614,7 @@
|
||||
"! rm -f custom.tar custom.tar.gz\n",
|
||||
"! tar cvf custom.tar custom\n",
|
||||
"! gzip custom.tar\n",
|
||||
"! gsutil cp custom.tar.gz $BUCKET_NAME/trainer_cifar10.tar.gz"
|
||||
]
|
||||
"! gcloud storage cp custom.tar.gz $BUCKET_NAME/trainer_cifar10.tar.gz" ]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
@@ -2062,8 +2058,7 @@
|
||||
" print(e)\n",
|
||||
"\n",
|
||||
"if delete_bucket and \"BUCKET_NAME\" in globals():\n",
|
||||
" ! gsutil rm -r $BUCKET_NAME"
|
||||
]
|
||||
" ! gcloud storage rm --recursive $BUCKET_NAME" ]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
|
||||
+6
-11
@@ -39,7 +39,7 @@
|
||||
" </td>\n",
|
||||
" <td>\n",
|
||||
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/gapic/custom/showcase_hyperparmeter_tuning_tabular_regression.ipynb\">\n",
|
||||
" <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\">\n",
|
||||
" <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\">\n",
|
||||
" View on GitHub\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
@@ -419,8 +419,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! gsutil mb -l $REGION $BUCKET_NAME"
|
||||
]
|
||||
"! gcloud storage buckets create --location $REGION $BUCKET_NAME" ]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
@@ -439,8 +438,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! gsutil ls -al $BUCKET_NAME"
|
||||
]
|
||||
"! gcloud storage ls --all-versions --long $BUCKET_NAME" ]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
@@ -1116,8 +1114,7 @@
|
||||
"! rm -f custom.tar custom.tar.gz\n",
|
||||
"! tar cvf custom.tar custom\n",
|
||||
"! gzip custom.tar\n",
|
||||
"! gsutil cp custom.tar.gz $BUCKET_NAME/trainer_boston.tar.gz"
|
||||
]
|
||||
"! gcloud storage cp custom.tar.gz $BUCKET_NAME/trainer_boston.tar.gz" ]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
@@ -1626,8 +1623,7 @@
|
||||
"! rm -f custom.tar custom.tar.gz\n",
|
||||
"! tar cvf custom.tar custom\n",
|
||||
"! gzip custom.tar\n",
|
||||
"! gsutil cp custom.tar.gz $BUCKET_NAME/trainer_boston.tar.gz"
|
||||
]
|
||||
"! gcloud storage cp custom.tar.gz $BUCKET_NAME/trainer_boston.tar.gz" ]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
@@ -2085,8 +2081,7 @@
|
||||
" print(e)\n",
|
||||
"\n",
|
||||
"if delete_bucket and \"BUCKET_NAME\" in globals():\n",
|
||||
" ! gsutil rm -r $BUCKET_NAME"
|
||||
]
|
||||
" ! gcloud storage rm --recursive $BUCKET_NAME" ]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user