xiaohei/model_garden
1
0
Fork 0
mirror of https://github.com/GoogleCloudPlatform/vertex-ai-samples.git synced 2026-09-26 14:42:04 +00:00
Code Issues Packages Projects Releases Wiki Activity

Migrate gsutil usage to gcloud storage (#4340)

Browse Source
This commit is contained in:
Margubur Rahman
2025-12-18 13:50:35 -05:00
committed by GitHub
parent f5d341ae45
commit 966bbc49a7
10 changed files with 63 additions and 124 deletions
Download Patch File Download Diff File Expand all files Collapse all files
notebooks/community/gapic/custom/showcase_hyperparmeter_tuning_image_classification.ipynb
+5 -10
View File
@@ -419,8 +419,7 @@
}, },
"outputs": [], "outputs": [],
"source": [ "source": [
"! gsutil mb -l $REGION $BUCKET_NAME" "! gcloud storage buckets create --location=$REGION $BUCKET_NAME" ]
]
}, },
{ {
"cell_type": "markdown", "cell_type": "markdown",
@@ -439,8 +438,7 @@
}, },
"outputs": [], "outputs": [],
"source": [ "source": [
"! gsutil ls -al $BUCKET_NAME" "! gcloud storage ls --all-versions --long $BUCKET_NAME" ]
]
}, },
{ {
"cell_type": "markdown", "cell_type": "markdown",
@@ -1119,8 +1117,7 @@
"! rm -f custom.tar custom.tar.gz\n", "! rm -f custom.tar custom.tar.gz\n",
"! tar cvf custom.tar custom\n", "! tar cvf custom.tar custom\n",
"! gzip custom.tar\n", "! gzip custom.tar\n",
"! gsutil cp custom.tar.gz $BUCKET_NAME/trainer_cifar10.tar.gz" "! gcloud storage cp custom.tar.gz $BUCKET_NAME/trainer_cifar10.tar.gz" ]
]
}, },
{ {
"cell_type": "markdown", "cell_type": "markdown",
@@ -1617,8 +1614,7 @@
"! rm -f custom.tar custom.tar.gz\n", "! rm -f custom.tar custom.tar.gz\n",
"! tar cvf custom.tar custom\n", "! tar cvf custom.tar custom\n",
"! gzip custom.tar\n", "! gzip custom.tar\n",
"! gsutil cp custom.tar.gz $BUCKET_NAME/trainer_cifar10.tar.gz" "! gcloud storage cp custom.tar.gz $BUCKET_NAME/trainer_cifar10.tar.gz" ]
]
}, },
{ {
"cell_type": "markdown", "cell_type": "markdown",
@@ -2062,8 +2058,7 @@
" print(e)\n", " print(e)\n",
"\n", "\n",
"if delete_bucket and \"BUCKET_NAME\" in globals():\n", "if delete_bucket and \"BUCKET_NAME\" in globals():\n",
" ! gsutil rm -r $BUCKET_NAME" " ! gcloud storage rm --recursive $BUCKET_NAME" ]
]
} }
], ],
"metadata": { "metadata": {
notebooks/community/gapic/custom/showcase_hyperparmeter_tuning_tabular_regression.ipynb
+5 -10
View File
@@ -419,8 +419,7 @@
}, },
"outputs": [], "outputs": [],
"source": [ "source": [
"! gsutil mb -l $REGION $BUCKET_NAME" "! gcloud storage buckets create --location $REGION $BUCKET_NAME" ]
]
}, },
{ {
"cell_type": "markdown", "cell_type": "markdown",
@@ -439,8 +438,7 @@
}, },
"outputs": [], "outputs": [],
"source": [ "source": [
"! gsutil ls -al $BUCKET_NAME" "! gcloud storage ls --all-versions --long $BUCKET_NAME" ]
]
}, },
{ {
"cell_type": "markdown", "cell_type": "markdown",
@@ -1116,8 +1114,7 @@
"! rm -f custom.tar custom.tar.gz\n", "! rm -f custom.tar custom.tar.gz\n",
"! tar cvf custom.tar custom\n", "! tar cvf custom.tar custom\n",
"! gzip custom.tar\n", "! gzip custom.tar\n",
"! gsutil cp custom.tar.gz $BUCKET_NAME/trainer_boston.tar.gz" "! gcloud storage cp custom.tar.gz $BUCKET_NAME/trainer_boston.tar.gz" ]
]
}, },
{ {
"cell_type": "markdown", "cell_type": "markdown",
@@ -1626,8 +1623,7 @@
"! rm -f custom.tar custom.tar.gz\n", "! rm -f custom.tar custom.tar.gz\n",
"! tar cvf custom.tar custom\n", "! tar cvf custom.tar custom\n",
"! gzip custom.tar\n", "! gzip custom.tar\n",
"! gsutil cp custom.tar.gz $BUCKET_NAME/trainer_boston.tar.gz" "! gcloud storage cp custom.tar.gz $BUCKET_NAME/trainer_boston.tar.gz" ]
]
}, },
{ {
"cell_type": "markdown", "cell_type": "markdown",
@@ -2085,8 +2081,7 @@
" print(e)\n", " print(e)\n",
"\n", "\n",
"if delete_bucket and \"BUCKET_NAME\" in globals():\n", "if delete_bucket and \"BUCKET_NAME\" in globals():\n",
" ! gsutil rm -r $BUCKET_NAME" " ! gcloud storage rm --recursive $BUCKET_NAME" ]
]
} }
], ],
"metadata": { "metadata": {
notebooks/community/gapic/custom/showcase_hyperparmeter_tuning_text_binary_classification.ipynb
+5 -10
View File
@@ -419,8 +419,7 @@
}, },
"outputs": [], "outputs": [],
"source": [ "source": [
"! gsutil mb -l $REGION $BUCKET_NAME" "! gcloud storage buckets create --location=$REGION $BUCKET_NAME" ]
]
}, },
{ {
"cell_type": "markdown", "cell_type": "markdown",
@@ -439,8 +438,7 @@
}, },
"outputs": [], "outputs": [],
"source": [ "source": [
"! gsutil ls -al $BUCKET_NAME" "! gcloud storage ls --all-versions --long $BUCKET_NAME" ]
]
}, },
{ {
"cell_type": "markdown", "cell_type": "markdown",
@@ -1116,8 +1114,7 @@
"! rm -f custom.tar custom.tar.gz\n", "! rm -f custom.tar custom.tar.gz\n",
"! tar cvf custom.tar custom\n", "! tar cvf custom.tar custom\n",
"! gzip custom.tar\n", "! gzip custom.tar\n",
"! gsutil cp custom.tar.gz $BUCKET_NAME/trainer_imdb.tar.gz" "! gcloud storage cp custom.tar.gz $BUCKET_NAME/trainer_imdb.tar.gz" ]
]
}, },
{ {
"cell_type": "markdown", "cell_type": "markdown",
@@ -1618,8 +1615,7 @@
"! rm -f custom.tar custom.tar.gz\n", "! rm -f custom.tar custom.tar.gz\n",
"! tar cvf custom.tar custom\n", "! tar cvf custom.tar custom\n",
"! gzip custom.tar\n", "! gzip custom.tar\n",
"! gsutil cp custom.tar.gz $BUCKET_NAME/trainer_imdb.tar.gz" "! gcloud storage cp custom.tar.gz $BUCKET_NAME/trainer_imdb.tar.gz" ]
]
}, },
{ {
"cell_type": "markdown", "cell_type": "markdown",
@@ -2060,8 +2056,7 @@
" print(e)\n", " print(e)\n",
"\n", "\n",
"if delete_bucket and \"BUCKET_NAME\" in globals():\n", "if delete_bucket and \"BUCKET_NAME\" in globals():\n",
" ! gsutil rm -r $BUCKET_NAME" " ! gcloud storage rm --recursive $BUCKET_NAME" ]
]
} }
], ],
"metadata": { "metadata": {
notebooks/community/migration/UJ10 Custom Training Prebuilt Container SKLearn.ipynb
+8 -16
View File
@@ -325,8 +325,7 @@
}, },
"outputs": [], "outputs": [],
"source": [ "source": [
"! gsutil mb -l $REGION gs://$BUCKET_NAME" "! gcloud storage buckets create --location=$REGION gs://$BUCKET_NAME" ]
]
}, },
{ {
"cell_type": "markdown", "cell_type": "markdown",
@@ -345,8 +344,7 @@
}, },
"outputs": [], "outputs": [],
"source": [ "source": [
"! gsutil ls -al gs://$BUCKET_NAME" "! gcloud storage ls --all-versions --long gs://$BUCKET_NAME" ]
]
}, },
{ {
"cell_type": "markdown", "cell_type": "markdown",
@@ -722,8 +720,7 @@
"! rm -f custom.tar custom.tar.gz\n", "! rm -f custom.tar custom.tar.gz\n",
"! tar cvf custom.tar custom\n", "! tar cvf custom.tar custom\n",
"! gzip custom.tar\n", "! gzip custom.tar\n",
"! gsutil cp custom.tar.gz gs://$BUCKET_NAME/census.tar.gz" "! gcloud storage cp custom.tar.gz gs://$BUCKET_NAME/census.tar.gz" ]
]
}, },
{ {
"cell_type": "markdown", "cell_type": "markdown",
@@ -1379,8 +1376,7 @@
" for i in INSTANCES:\n", " for i in INSTANCES:\n",
" f.write(json.dumps(i) + \"\\n\")\n", " f.write(json.dumps(i) + \"\\n\")\n",
"\n", "\n",
"! gsutil cat $gcs_input_uri" "! gcloud storage cat $gcs_input_uri" ]
]
}, },
{ {
"cell_type": "markdown", "cell_type": "markdown",
@@ -1718,8 +1714,7 @@
"source": [ "source": [
"def get_latest_predictions(gcs_out_dir):\n", "def get_latest_predictions(gcs_out_dir):\n",
" \"\"\" Get the latest prediction subfolder using the timestamp in the subfolder name\"\"\"\n", " \"\"\" Get the latest prediction subfolder using the timestamp in the subfolder name\"\"\"\n",
" folders = !gsutil ls $gcs_out_dir\n", " folders = !gcloud storage ls $gcs_out_dir\n", " latest = \"\"\n",
" latest = \"\"\n",
" for folder in folders:\n", " for folder in folders:\n",
" subfolder = folder.split(\"/\")[-2]\n", " subfolder = folder.split(\"/\")[-2]\n",
" if subfolder.startswith(\"prediction-\"):\n", " if subfolder.startswith(\"prediction-\"):\n",
@@ -1738,10 +1733,8 @@
" folder = get_latest_predictions(\n", " folder = get_latest_predictions(\n",
" response.output_config.gcs_destination.output_uri_prefix\n", " response.output_config.gcs_destination.output_uri_prefix\n",
" )\n", " )\n",
" ! gsutil ls $folder/prediction*\n", " ! gcloud storage ls $folder/prediction*\n", "\n",
"\n", " ! gcloud storage cat --display-url $folder/prediction*\n", " break\n",
" ! gsutil cat -h $folder/prediction*\n",
" break\n",
" time.sleep(60)" " time.sleep(60)"
] ]
}, },
@@ -2630,8 +2623,7 @@
" print(e)\n", " print(e)\n",
"\n", "\n",
"if delete_bucket and \"BUCKET_NAME\" in globals():\n", "if delete_bucket and \"BUCKET_NAME\" in globals():\n",
" ! gsutil rm -r gs://$BUCKET_NAME" " ! gcloud storage rm --recursive gs://$BUCKET_NAME" ]
]
} }
], ],
"metadata": { "metadata": {
notebooks/community/migration/UJ15 AutoML for vision with Vertex AI Video Object Tracking.ipynb
+9 -18
View File
@@ -325,8 +325,7 @@
}, },
"outputs": [], "outputs": [],
"source": [ "source": [
"! gsutil mb -l $REGION gs://$BUCKET_NAME" "! gcloud storage buckets create --location=$REGION gs://$BUCKET_NAME" ]
]
}, },
{ {
"cell_type": "markdown", "cell_type": "markdown",
@@ -345,8 +344,7 @@
}, },
"outputs": [], "outputs": [],
"source": [ "source": [
"! gsutil ls -al gs://$BUCKET_NAME" "! gcloud storage ls --all-versions --long gs://$BUCKET_NAME" ]
]
}, },
{ {
"cell_type": "markdown", "cell_type": "markdown",
@@ -542,8 +540,7 @@
}, },
"outputs": [], "outputs": [],
"source": [ "source": [
"! gsutil cat $IMPORT_FILE | head -n 10" "! gcloud storage cat $IMPORT_FILE | head -n 10" ]
]
}, },
{ {
"cell_type": "markdown", "cell_type": "markdown",
@@ -1386,8 +1383,7 @@
}, },
"outputs": [], "outputs": [],
"source": [ "source": [
"test_items = ! gsutil cat $IMPORT_FILE | head -n25\n", "test_items = ! gcloud storage cat $IMPORT_FILE | head -n25\n", "\n",
"\n",
"cols_1 = test_items[0].split(\",\")\n", "cols_1 = test_items[0].split(\",\")\n",
"cols_2 = test_items[-1].split(\",\")\n", "cols_2 = test_items[-1].split(\",\")\n",
"\n", "\n",
@@ -1466,8 +1462,7 @@
" f.write(json.dumps(data) + \"\\n\")\n", " f.write(json.dumps(data) + \"\\n\")\n",
"\n", "\n",
"print(gcs_input_uri)\n", "print(gcs_input_uri)\n",
"!gsutil cat $gcs_input_uri" "!gcloud storage cat $gcs_input_uri" ]
]
}, },
{ {
"cell_type": "markdown", "cell_type": "markdown",
@@ -1783,8 +1778,7 @@
"source": [ "source": [
"def get_latest_predictions(gcs_out_dir):\n", "def get_latest_predictions(gcs_out_dir):\n",
" \"\"\" Get the latest prediction subfolder using the timestamp in the subfolder name\"\"\"\n", " \"\"\" Get the latest prediction subfolder using the timestamp in the subfolder name\"\"\"\n",
" folders = !gsutil ls $gcs_out_dir\n", " folders = !gcloud storage ls $gcs_out_dir\n", " latest = \"\"\n",
" latest = \"\"\n",
" for folder in folders:\n", " for folder in folders:\n",
" subfolder = folder.split(\"/\")[-2]\n", " subfolder = folder.split(\"/\")[-2]\n",
" if subfolder.startswith(\"prediction-\"):\n", " if subfolder.startswith(\"prediction-\"):\n",
@@ -1803,10 +1797,8 @@
" folder = get_latest_predictions(\n", " folder = get_latest_predictions(\n",
" response.output_config.gcs_destination.output_uri_prefix\n", " response.output_config.gcs_destination.output_uri_prefix\n",
" )\n", " )\n",
" ! gsutil ls $folder/prediction**\n", " ! gcloud storage ls $folder/prediction**\n", "\n",
"\n", " ! gcloud storage cat $folder/prediction**\n", " break\n",
" ! gsutil cat $folder/prediction**\n",
" break\n",
" time.sleep(60)" " time.sleep(60)"
] ]
}, },
@@ -1882,8 +1874,7 @@
" print(e)\n", " print(e)\n",
"\n", "\n",
"if delete_bucket and \"BUCKET_NAME\" in globals():\n", "if delete_bucket and \"BUCKET_NAME\" in globals():\n",
" ! gsutil rm -r gs://$BUCKET_NAME" " ! gcloud storage rm --recursive gs://$BUCKET_NAME" ]
]
} }
], ],
"metadata": { "metadata": {
notebooks/community/migration/UJ2,12 Custom Training Prebuilt Container TF Keras.ipynb
+8 -16
View File
@@ -325,8 +325,7 @@
}, },
"outputs": [], "outputs": [],
"source": [ "source": [
"! gsutil mb -l $REGION gs://$BUCKET_NAME" "! gcloud storage buckets create --location $REGION gs://$BUCKET_NAME" ]
]
}, },
{ {
"cell_type": "markdown", "cell_type": "markdown",
@@ -345,8 +344,7 @@
}, },
"outputs": [], "outputs": [],
"source": [ "source": [
"! gsutil ls -al gs://$BUCKET_NAME" "! gcloud storage ls --all-versions --long gs://$BUCKET_NAME" ]
]
}, },
{ {
"cell_type": "markdown", "cell_type": "markdown",
@@ -659,8 +657,7 @@
"! rm -f cifar.tar cifar.tar.gz\n", "! rm -f cifar.tar cifar.tar.gz\n",
"! tar cvf cifar.tar cifar\n", "! tar cvf cifar.tar cifar\n",
"! gzip cifar.tar\n", "! gzip cifar.tar\n",
"! gsutil cp cifar.tar.gz gs://$BUCKET_NAME/trainer_cifar.tar.gz" "! gcloud storage cp cifar.tar.gz gs://$BUCKET_NAME/trainer_cifar.tar.gz" ]
]
}, },
{ {
"cell_type": "markdown", "cell_type": "markdown",
@@ -1318,8 +1315,7 @@
" b64str = base64.b64encode(bytes.numpy()).decode(\"utf-8\")\n", " b64str = base64.b64encode(bytes.numpy()).decode(\"utf-8\")\n",
" f.write(json.dumps({input_name: {\"b64\": b64str}}) + \"\\n\")\n", " f.write(json.dumps({input_name: {\"b64\": b64str}}) + \"\\n\")\n",
"\n", "\n",
"! gsutil cat $gcs_input_uri" "! gcloud storage cat $gcs_input_uri" ]
]
}, },
{ {
"cell_type": "markdown", "cell_type": "markdown",
@@ -1643,8 +1639,7 @@
"source": [ "source": [
"def get_latest_predictions(gcs_out_dir):\n", "def get_latest_predictions(gcs_out_dir):\n",
" \"\"\" Get the latest prediction subfolder using the timestamp in the subfolder name\"\"\"\n", " \"\"\" Get the latest prediction subfolder using the timestamp in the subfolder name\"\"\"\n",
" folders = !gsutil ls $gcs_out_dir\n", " folders = !gcloud storage ls $gcs_out_dir\n", " latest = \"\"\n",
" latest = \"\"\n",
" for folder in folders:\n", " for folder in folders:\n",
" subfolder = folder.split(\"/\")[-2]\n", " subfolder = folder.split(\"/\")[-2]\n",
" if subfolder.startswith(\"prediction-\"):\n", " if subfolder.startswith(\"prediction-\"):\n",
@@ -1663,10 +1658,8 @@
" folder = get_latest_predictions(\n", " folder = get_latest_predictions(\n",
" response.output_config.gcs_destination.output_uri_prefix\n", " response.output_config.gcs_destination.output_uri_prefix\n",
" )\n", " )\n",
" ! gsutil ls $folder/prediction*\n", " ! gcloud storage ls $folder/prediction*\n", "\n",
"\n", " ! gcloud storage cat $folder/prediction*\n", " break\n",
" ! gsutil cat $folder/prediction*\n",
" break\n",
" time.sleep(60)" " time.sleep(60)"
] ]
}, },
@@ -2234,8 +2227,7 @@
" print(e)\n", " print(e)\n",
"\n", "\n",
"if delete_bucket and \"BUCKET_NAME\" in globals():\n", "if delete_bucket and \"BUCKET_NAME\" in globals():\n",
" ! gsutil rm -r gs://$BUCKET_NAME" " ! gcloud storage rm --recursive gs://$BUCKET_NAME" ]
]
} }
], ],
"metadata": { "metadata": {
notebooks/community/migration/UJ3 Custom Training Custom Container TF Keras.ipynb
+8 -17
View File
@@ -325,8 +325,7 @@
}, },
"outputs": [], "outputs": [],
"source": [ "source": [
"! gsutil mb -l $REGION gs://$BUCKET_NAME" "! gcloud storage buckets create --location=$REGION gs://$BUCKET_NAME" ]
]
}, },
{ {
"cell_type": "markdown", "cell_type": "markdown",
@@ -345,8 +344,7 @@
}, },
"outputs": [], "outputs": [],
"source": [ "source": [
"! gsutil ls -al gs://$BUCKET_NAME" "! gcloud storage ls --all-versions --long gs://$BUCKET_NAME" ]
]
}, },
{ {
"cell_type": "markdown", "cell_type": "markdown",
@@ -1338,9 +1336,7 @@
"cv2.imwrite(\"tmp1.jpg\", (test_image_1 * 255).astype(np.uint8))\n", "cv2.imwrite(\"tmp1.jpg\", (test_image_1 * 255).astype(np.uint8))\n",
"cv2.imwrite(\"tmp2.jpg\", (test_image_2 * 255).astype(np.uint8))\n", "cv2.imwrite(\"tmp2.jpg\", (test_image_2 * 255).astype(np.uint8))\n",
"\n", "\n",
"! gsutil cp tmp1.jpg gs://$BUCKET_NAME/tmp1.jpg\n", "! gcloud storage cp tmp1.jpg gs://$BUCKET_NAME/tmp1.jpg\n", "! gcloud storage cp tmp2.jpg gs://$BUCKET_NAME/tmp2.jpg\n", "\n",
"! gsutil cp tmp2.jpg gs://$BUCKET_NAME/tmp2.jpg\n",
"\n",
"test_item_1 = \"gs://\" + BUCKET_NAME + \"/\" + \"tmp1.jpg\"\n", "test_item_1 = \"gs://\" + BUCKET_NAME + \"/\" + \"tmp1.jpg\"\n",
"test_item_2 = \"gs://\" + BUCKET_NAME + \"/\" + \"tmp2.jpg\"" "test_item_2 = \"gs://\" + BUCKET_NAME + \"/\" + \"tmp2.jpg\""
] ]
@@ -1382,8 +1378,7 @@
" data = {input_name: {\"b64\": b64str}}\n", " data = {input_name: {\"b64\": b64str}}\n",
" f.write(json.dumps(data) + \"\\n\")\n", " f.write(json.dumps(data) + \"\\n\")\n",
"\n", "\n",
"! gsutil cat $gcs_input_uri" "! gcloud storage cat $gcs_input_uri" ]
]
}, },
{ {
"cell_type": "markdown", "cell_type": "markdown",
@@ -1706,8 +1701,7 @@
"source": [ "source": [
"def get_latest_predictions(gcs_out_dir):\n", "def get_latest_predictions(gcs_out_dir):\n",
" \"\"\" Get the latest prediction subfolder using the timestamp in the subfolder name\"\"\"\n", " \"\"\" Get the latest prediction subfolder using the timestamp in the subfolder name\"\"\"\n",
" folders = !gsutil ls $gcs_out_dir\n", " folders = !gcloud storage ls $gcs_out_dir\n", " latest = \"\"\n",
" latest = \"\"\n",
" for folder in folders:\n", " for folder in folders:\n",
" subfolder = folder.split(\"/\")[-2]\n", " subfolder = folder.split(\"/\")[-2]\n",
" if subfolder.startswith(\"prediction-\"):\n", " if subfolder.startswith(\"prediction-\"):\n",
@@ -1726,10 +1720,8 @@
" folder = get_latest_predictions(\n", " folder = get_latest_predictions(\n",
" response.output_config.gcs_destination.output_uri_prefix\n", " response.output_config.gcs_destination.output_uri_prefix\n",
" )\n", " )\n",
" ! gsutil ls $folder/prediction*\n", ! gcloud storage ls $folder/prediction*\n", "\n",
"\n", ! gcloud storage cat $folder/prediction*\n", " break\n",
" ! gsutil cat $folder/prediction*\n",
" break\n",
" time.sleep(60)" " time.sleep(60)"
] ]
}, },
@@ -2306,8 +2298,7 @@
" print(e)\n", " print(e)\n",
"\n", "\n",
"if delete_bucket and \"BUCKET_NAME\" in globals():\n", "if delete_bucket and \"BUCKET_NAME\" in globals():\n",
" ! gsutil rm -r gs://$BUCKET_NAME" " ! gcloud storage rm --recursive gs://$BUCKET_NAME" ]
]
} }
], ],
"metadata": { "metadata": {
notebooks/community/ml_ops/stage3/get_started_with_rapid_prototyping_bqml_automl.ipynb
+6 -12
View File
@@ -550,8 +550,7 @@
}, },
"outputs": [], "outputs": [],
"source": [ "source": [
"! gsutil mb -l $REGION $BUCKET_URI" "! gcloud storage buckets create --location $REGION $BUCKET_URI" ]
]
}, },
{ {
"cell_type": "markdown", "cell_type": "markdown",
@@ -570,8 +569,7 @@
}, },
"outputs": [], "outputs": [],
"source": [ "source": [
"! gsutil ls -al $BUCKET_URI" "! gcloud storage ls --all-versions --long $BUCKET_URI" ]
]
}, },
{ {
"cell_type": "markdown", "cell_type": "markdown",
@@ -641,10 +639,8 @@
}, },
"outputs": [], "outputs": [],
"source": [ "source": [
"! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectCreator $BUCKET_URI\n", "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n", "\n",
"\n", "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer" ]
"! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectViewer $BUCKET_URI"
]
}, },
{ {
"cell_type": "markdown", "cell_type": "markdown",
@@ -722,8 +718,7 @@
"\n", "\n",
"RAW_INPUT_DATA = f\"{DATA_FOLDER}/abalone.csv\"\n", "RAW_INPUT_DATA = f\"{DATA_FOLDER}/abalone.csv\"\n",
"\n", "\n",
"! gsutil cp gs://cloud-samples-data/vertex-ai/community-content/datasets/abalone/abalone.data {RAW_INPUT_DATA}" "! gcloud storage cp gs://cloud-samples-data/vertex-ai/community-content/datasets/abalone/abalone.data {RAW_INPUT_DATA}" ]
]
}, },
{ {
"cell_type": "markdown", "cell_type": "markdown",
@@ -1838,8 +1833,7 @@
"\n", "\n",
"\n", "\n",
"if delete_bucket or os.getenv(\"IS_TESTING\"):\n", "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
" ! gsutil rm -r $BUCKET_URI" " ! gcloud storage rm --recursive $BUCKET_URI" ]
]
} }
], ],
"metadata": { "metadata": {
notebooks/community/model_garden/model_garden_advanced_features.ipynb
+6 -8
View File
@@ -186,11 +186,10 @@
"if BUCKET_URI is None or BUCKET_URI.strip() == \"\" or BUCKET_URI == \"gs://\":\n", "if BUCKET_URI is None or BUCKET_URI.strip() == \"\" or BUCKET_URI == \"gs://\":\n",
" BUCKET_URI = f\"gs://{PROJECT_ID}-tmp-{now}-{str(uuid.uuid4())[:4]}\"\n", " BUCKET_URI = f\"gs://{PROJECT_ID}-tmp-{now}-{str(uuid.uuid4())[:4]}\"\n",
" BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n", " BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n",
" ! gsutil mb -l {REGION} {BUCKET_URI}\n", " ! gcloud storage buckets create --location={REGION} {BUCKET_URI}\n", "else:\n",
"else:\n",
" assert BUCKET_URI.startswith(\"gs://\"), \"BUCKET_URI must start with `gs://`.\"\n", " assert BUCKET_URI.startswith(\"gs://\"), \"BUCKET_URI must start with `gs://`.\"\n",
" shell_output = ! gsutil ls -Lb {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n", "# Note: The format of the full listing output is different. gcloud storage uses a title case for keys and will not display a field if its value is \"None\".\n",
" bucket_region = shell_output[0].strip().lower()\n", " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n", " bucket_region = shell_output[0].strip().lower()\n",
" if bucket_region != REGION:\n", " if bucket_region != REGION:\n",
" raise ValueError(\n", " raise ValueError(\n",
" \"Bucket region %s is different from notebook region %s\"\n", " \"Bucket region %s is different from notebook region %s\"\n",
@@ -214,8 +213,8 @@
"\n", "\n",
"\n", "\n",
"# Provision permissions to the SERVICE_ACCOUNT with the GCS bucket\n", "# Provision permissions to the SERVICE_ACCOUNT with the GCS bucket\n",
"! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.admin $BUCKET_NAME\n", "! # Note: Migrating scripts using gsutil iam ch is more complex than get or set. You need to replace the single iam ch command with a series of gcloud storage bucket add-iam-policy-binding and/or gcloud storage bucket remove-iam-policy-binding commands, or replicate the read-modify-write loop.\n",
"\n", "! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=\"serviceAccount:{SERVICE_ACCOUNT}\" --role=\"roles/storage.admin\"\n", "\n",
"! gcloud config set project $PROJECT_ID\n", "! gcloud config set project $PROJECT_ID\n",
"! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/storage.admin\"\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/storage.admin\"\n",
"! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/aiplatform.user\"" "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/aiplatform.user\""
@@ -1593,8 +1592,7 @@
"\n", "\n",
"delete_bucket = False # @param {type:\"boolean\"}\n", "delete_bucket = False # @param {type:\"boolean\"}\n",
"if delete_bucket:\n", "if delete_bucket:\n",
" ! gsutil -m rm -r $BUCKET_NAME" " ! gcloud storage rm --recursive $BUCKET_NAME" ]
]
} }
], ],
"metadata": { "metadata": {
notebooks/official/automl/automl_forecasting_on_vertex_pipelines.ipynb
+3 -7
View File
@@ -344,8 +344,7 @@
}, },
"outputs": [], "outputs": [],
"source": [ "source": [
"! gsutil mb -l {LOCATION} -p {PROJECT_ID} {BUCKET_URI}" "! gcloud storage buckets create --location {LOCATION} --project {PROJECT_ID} {BUCKET_URI}" ]
]
}, },
{ {
"cell_type": "markdown", "cell_type": "markdown",
@@ -408,9 +407,7 @@
}, },
"outputs": [], "outputs": [],
"source": [ "source": [
"! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectCreator $BUCKET_URI\n", "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n", "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer" ]
"! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectViewer $BUCKET_URI"
]
}, },
{ {
"cell_type": "markdown", "cell_type": "markdown",
@@ -1782,8 +1779,7 @@
"# Delete Cloud Storage objects that were created\n", "# Delete Cloud Storage objects that were created\n",
"delete_bucket = True\n", "delete_bucket = True\n",
"if delete_bucket:\n", "if delete_bucket:\n",
" ! gsutil -m rm -r $BUCKET_URI" " ! gcloud storage rm --recursive $BUCKET_URI" ]
]
} }
], ],
"metadata": { "metadata": {