mirror of
https://github.com/GoogleCloudPlatform/vertex-ai-samples.git
synced 2026-09-26 14:42:04 +00:00
Migrate gsutil usage to gcloud storage (#4298)
* Migrate gsutil usage to gcloud storage
* update
* linter changes for 4298
* Revert "linter changes for 4298"
This reverts commit c7a00a9710.
* Linter fixes
* update
* update
* Update distributed_hyperparameter_tuning.ipynb
* removed changes in model_garden folder
---------
Co-authored-by: gurusai-voleti <gvoleti@google.com>
This commit is contained in:
Margubur Rahman
co-authored by
gurusai-voleti
gurusai-voleti
parent
5efa51206f
commit
5b6c766629
@@ -153,8 +153,8 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"!gsutil cp gs://visionai-artifacts/visionai-0.0.6-py3-none-any.whl .\n",
|
||||
"!pip install visionai-0.0.6-py3-none-any.whl --force-reinstall"
|
||||
"! gcloud storage cp gs://visionai-artifacts/visionai-0.0.6-py3-none-any.whl .\n",
|
||||
"! pip install visionai-0.0.6-py3-none-any.whl --force-reinstall"
|
||||
]
|
||||
},
|
||||
{
|
||||
|
||||
@@ -286,7 +286,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! gsutil mb -l $LOCATION -p $PROJECT_ID $BUCKET_URI"
|
||||
"! gcloud storage buckets create --location $LOCATION --project $PROJECT_ID $BUCKET_URI"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -770,7 +770,7 @@
|
||||
"delete_bucket = False # Set True for deletion\n",
|
||||
"\n",
|
||||
"if delete_bucket:\n",
|
||||
" ! gsutil rm -rf {BUCKET_URI}"
|
||||
" ! gcloud storage rm --recursive --continue-on-error {BUCKET_URI}"
|
||||
]
|
||||
}
|
||||
],
|
||||
|
||||
+5
-5
@@ -300,7 +300,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! gsutil mb -l $LOCATION -p $PROJECT_ID $BUCKET_URI"
|
||||
"! gcloud storage buckets create --location $LOCATION --project $PROJECT_ID $BUCKET_URI"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -430,9 +430,9 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectCreator $BUCKET_URI\n",
|
||||
"! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n",
|
||||
"\n",
|
||||
"! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectViewer $BUCKET_URI"
|
||||
"! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -482,7 +482,7 @@
|
||||
"SOURCE_DATA_URL = \"gs://cloud-samples-data/vertex-ai/dataset-management/datasets/uci_glass_preprocessed/glass.csv\"\n",
|
||||
"DESTINATION_DATA_URL = f\"{BUCKET_URI}/data/glass.csv\"\n",
|
||||
"\n",
|
||||
"! gsutil cp $SOURCE_DATA_URL $DESTINATION_DATA_URL"
|
||||
"! gcloud storage cp $SOURCE_DATA_URL $DESTINATION_DATA_URL"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -807,7 +807,7 @@
|
||||
"\n",
|
||||
"# Delete Cloud Storage objects that were created\n",
|
||||
"if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
|
||||
" ! gsutil -m rm -r $BUCKET_URI"
|
||||
" ! gcloud storage rm --recursive $BUCKET_URI"
|
||||
]
|
||||
}
|
||||
],
|
||||
|
||||
+8
-8
@@ -292,7 +292,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! gsutil mb -l $LOCATION -p $PROJECT_ID $BUCKET_URI"
|
||||
"! gcloud storage buckets create --location=$LOCATION --project=$PROJECT_ID $BUCKET_URI"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -371,13 +371,13 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"count = ! gsutil cat $IMPORT_FILE | wc -l\n",
|
||||
"count = ! gcloud storage cat $IMPORT_FILE | wc -l\n",
|
||||
"print(\"Number of Examples\", int(count[0]))\n",
|
||||
"\n",
|
||||
"print(\"First 10 rows\")\n",
|
||||
"! gsutil cat $IMPORT_FILE | head\n",
|
||||
"! gcloud storage cat $IMPORT_FILE | head\n",
|
||||
"\n",
|
||||
"heading = ! gsutil cat $IMPORT_FILE | head -n1\n",
|
||||
"heading = ! gcloud storage cat $IMPORT_FILE | head -n1\n",
|
||||
"label_column = str(heading).split(\",\")[-1].split(\"'\")[0]\n",
|
||||
"print(\"Label Column Name\", label_column)\n",
|
||||
"if label_column is None:\n",
|
||||
@@ -580,14 +580,14 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! gsutil cat $IMPORT_FILE | head -n 1 > tmp.csv\n",
|
||||
"! gsutil cat $IMPORT_FILE | tail -n 10 >> tmp.csv\n",
|
||||
"! gcloud storage cat $IMPORT_FILE | head -n 1 > tmp.csv\n",
|
||||
"! gcloud storage cat $IMPORT_FILE | tail -n 10 >> tmp.csv\n",
|
||||
"\n",
|
||||
"! cut -d, -f1-16 tmp.csv > batch.csv\n",
|
||||
"\n",
|
||||
"gcs_input_uri = BUCKET_URI + \"/test.csv\"\n",
|
||||
"\n",
|
||||
"! gsutil cp batch.csv $gcs_input_uri"
|
||||
"! gcloud storage cp batch.csv $gcs_input_uri"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -729,7 +729,7 @@
|
||||
"delete_bucket = False\n",
|
||||
"\n",
|
||||
"if delete_bucket:\n",
|
||||
" ! gsutil rm -r $BUCKET_URI\n",
|
||||
" ! gcloud storage rm --recursive $BUCKET_URI\n",
|
||||
"\n",
|
||||
"# Delete the locally generated files\n",
|
||||
"! rm batch.csv tmp.csv"
|
||||
|
||||
+7
-5
@@ -332,7 +332,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! gsutil mb -l $LOCATION -p $PROJECT_ID $BUCKET_URI"
|
||||
"! gcloud storage buckets create --location=$LOCATION --project=$PROJECT_ID $BUCKET_URI"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -405,9 +405,9 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectCreator $BUCKET_URI\n",
|
||||
"! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n",
|
||||
"\n",
|
||||
"! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectViewer $BUCKET_URI"
|
||||
"! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -506,7 +506,9 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"hp_dict: str = '{\"num_hidden_layers\": 3, \"hidden_size\": 32, \"learning_rate\": 0.01, \"epochs\": 1, \"steps_per_epoch\": -1}'\n",
|
||||
"hp_dict: str = (\n",
|
||||
" '{\"num_hidden_layers\": 3, \"hidden_size\": 32, \"learning_rate\": 0.01, \"epochs\": 1, \"steps_per_epoch\": -1}'\n",
|
||||
")\n",
|
||||
"data_dir: str = (\n",
|
||||
" \"gs://cloud-samples-data/vertex-ai/pipeline-deployment/datasets/bikes_weather/\"\n",
|
||||
")\n",
|
||||
@@ -732,7 +734,7 @@
|
||||
"source": [
|
||||
"delete_bucket = False\n",
|
||||
"if delete_bucket:\n",
|
||||
" ! gsutil rm -r $BUCKET_URI"
|
||||
" ! gcloud storage rm --recursive $BUCKET_URI"
|
||||
]
|
||||
}
|
||||
],
|
||||
|
||||
+5
-5
@@ -303,7 +303,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! gsutil mb -l {LOCATION} -p {PROJECT_ID} {BUCKET_URI}"
|
||||
"! gcloud storage buckets create --location={LOCATION} --project={PROJECT_ID} {BUCKET_URI}"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -545,9 +545,9 @@
|
||||
"source": [
|
||||
"MODEL_URI = f\"{BUCKET_URI}/{model_name}\"\n",
|
||||
"\n",
|
||||
"!gsutil rm -r $MODEL_URI\n",
|
||||
"!gsutil cp -r $model_path $MODEL_URI\n",
|
||||
"!gsutil ls -al $MODEL_URI"
|
||||
"!gcloud storage rm --recursive $MODEL_URI\n",
|
||||
"!gcloud storage cp --recursive $model_path $MODEL_URI\n",
|
||||
"!gcloud storage ls --all-versions --long $MODEL_URI"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -823,7 +823,7 @@
|
||||
"\n",
|
||||
"delete_bucket = False\n",
|
||||
"if delete_bucket:\n",
|
||||
" ! gsutil -m rm -r $BUCKET_URI"
|
||||
" ! gcloud storage rm --recursive $BUCKET_URI"
|
||||
]
|
||||
}
|
||||
],
|
||||
|
||||
@@ -274,7 +274,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! gsutil mb -l {LOCATION} {BUCKET_URI}"
|
||||
"! gcloud storage buckets create --location={LOCATION} {BUCKET_URI}"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -692,7 +692,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Set this to true only if you'd like to delete your bucket\n",
|
||||
"# Set this to true only if you'd like to delete your bucket.\n",
|
||||
"delete_bucket = False\n",
|
||||
"\n",
|
||||
"hp_job.delete()\n",
|
||||
@@ -700,7 +700,7 @@
|
||||
"!gcloud container images delete $IMAGE_URI --force-delete-tags --quiet\n",
|
||||
"\n",
|
||||
"if delete_bucket:\n",
|
||||
" ! gsutil rm -r $BUCKET_URI"
|
||||
" ! gcloud storage rm --recursive $BUCKET_URI"
|
||||
]
|
||||
}
|
||||
],
|
||||
|
||||
Reference in New Issue
Block a user