xiaohei/model_garden
1
0
Fork 0
mirror of https://github.com/GoogleCloudPlatform/vertex-ai-samples.git synced 2026-09-26 14:42:04 +00:00
Code Issues Packages Projects Releases Wiki Activity

rebrand and clean (#2350)

Browse Source 
* rebrand and clean

* fix: Added better type checking and fixed misspelling of _GCP_VPC_NETWORK_NAME

---------

Co-authored-by: ivanmkc@google.com <ivanmkc@google.com>
This commit is contained in:
Andrew Ferlitsch
2023-10-20 15:50:52 +00:00
committed by GitHub
co-authored by ivanmkc@google.com <ivanmkc@google.com>
parent f0dacd8ccd
commit 1cab2c2d1c
4 changed files with 62 additions and 89 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.cloud-build/execute_changed_notebooks_helper.py
+41 -36
View File
@@ -89,11 +89,10 @@ class NotebookExecutionResult:
return None
def load_results(results_bucket: str,
results_file: str) -> Dict[str, Any]:
'''
def load_results(results_bucket: str, results_file: str) -> Dict[str, Any]:
"""
Load accumulated notebook test results
'''
"""
print("Loading existing accumulative results ...")
accumulative_results = {}
@@ -105,10 +104,14 @@ def load_results(results_bucket: str,
blobs = client.list_blobs(results_bucket, prefix=build_results_dir)
for blob in blobs:
time_created = blob.time_created.replace(tzinfo=None)
if (datetime.datetime.now().replace(tzinfo=None) - time_created).total_seconds() > MAX_RESULTS_AGE_SECONDS:
if (
datetime.datetime.now().replace(tzinfo=None) - time_created
).total_seconds() > MAX_RESULTS_AGE_SECONDS:
continue
content = util.download_blob_into_memory(results_bucket, blob.name, download_as_text=True)
content = util.download_blob_into_memory(
results_bucket, blob.name, download_as_text=True
)
try:
build_results = json.loads(content)
@@ -116,8 +119,12 @@ def load_results(results_bucket: str,
continue # skip corrupted build results files
for notebook in build_results:
if notebook in accumulative_results:
accumulative_results[notebook]['passed'] += build_results[notebook]['passed']
accumulative_results[notebook]['failed'] += build_results[notebook]['failed']
accumulative_results[notebook]["passed"] += build_results[notebook][
"passed"
]
accumulative_results[notebook]["failed"] += build_results[notebook][
"failed"
]
else:
accumulative_results[notebook] = build_results[notebook]
@@ -128,16 +135,17 @@ def load_results(results_bucket: str,
# If there are no accumulative results, an empty dict is returned
return accumulative_results
def select_notebook(changed_notebook: str,
accumulative_results: Dict[str, Any],
test_percent: int) -> bool:
'''
def select_notebook(
changed_notebook: str, accumulative_results: Dict[str, Any], test_percent: int
) -> bool:
"""
Algorithm to randomly select a notebook, but weight the propbability of selected based on past failures
'''
"""
if changed_notebook in accumulative_results:
pass_count = accumulative_results[changed_notebook]['passed']
fail_count = accumulative_results[changed_notebook]['failed']
pass_count = accumulative_results[changed_notebook]["passed"]
fail_count = accumulative_results[changed_notebook]["failed"]
else:
pass_count = 1
fail_count = 0
@@ -151,7 +159,9 @@ def select_notebook(changed_notebook: str,
should_test_due_to_random_subset = random.uniform(0, 1) <= (test_percent / 100)
if should_test_due_to_failure or should_test_due_to_random_subset:
print(f"Selected: {changed_notebook}, {should_test_due_to_failure}, {should_test_due_to_random_subset}")
print(
f"Selected: {changed_notebook}, {should_test_due_to_failure}, {should_test_due_to_random_subset}"
)
return True
else:
print(f"Not Selected: {changed_notebook}, pass {pass_count}, fail {fail_count}")
@@ -176,7 +186,7 @@ def _process_notebook(
"PROJECT_ID": variable_project_id,
"REGION": variable_region,
"SERVICE_ACCOUNT": variable_service_account,
"VPC_NETWORK": variable_vpc_network,
"VPC_NETWORK": variable_vpc_network or "",
},
)
unique_strings_preprocessor = NotebookProcessors.UniqueStringsPreprocessor()
@@ -233,8 +243,6 @@ def _create_tag(filepath: str) -> str:
return tag
def process_and_execute_notebook(
container_uri: str,
staging_bucket: str,
@@ -248,7 +256,6 @@ def process_and_execute_notebook(
notebook: str,
should_get_tail_logs: bool = False,
) -> NotebookExecutionResult:
print(f"Running notebook: {notebook}")
# Handle empty strings
@@ -413,11 +420,11 @@ def get_changed_notebooks(
return notebooks
def _save_results(results: List[NotebookExecutionResult],
artifacts_bucket: str,
results_file: str):
artifacts_bucket = artifacts_bucket.replace("gs://", "").split('/')[0]
def _save_results(
results: List[NotebookExecutionResult], artifacts_bucket: str, results_file: str
):
artifacts_bucket = artifacts_bucket.replace("gs://", "").split("/")[0]
print("Updating build results ...")
build_results = {}
@@ -429,10 +436,10 @@ def _save_results(results: List[NotebookExecutionResult],
pass_count = 0
fail_count = 1
build_results[result.path] = {
'duration': result.duration.total_seconds(),
'start_time': str(result.start_time),
'passed': pass_count,
'failed': fail_count
"duration": result.duration.total_seconds(),
"start_time": str(result.start_time),
"passed": pass_count,
"failed": fail_count,
}
print(f"adding {result.path}")
@@ -441,8 +448,7 @@ def _save_results(results: List[NotebookExecutionResult],
client = storage.Client()
bucket = client.get_bucket(artifacts_bucket)
bucket.blob(str(results_file)).upload_from_string(content, 'text/json')
bucket.blob(str(results_file)).upload_from_string(content, "text/json")
def process_and_execute_notebooks(
@@ -480,7 +486,7 @@ def process_and_execute_notebooks(
artifacts_bucket (str):
Required. The GCS staging bucket to write executed notebooks to.
results_file (str):
Required: The path to the artifacts bucket to save results
Required: The path to the artifacts bucket to save results
variable_project_id (str):
Required. The value for PROJECT_ID to inject into notebooks.
variable_region (str):
@@ -507,8 +513,9 @@ def process_and_execute_notebooks(
"Running notebooks in parallel, so no logs will be displayed. Please wait..."
)
with concurrent.futures.ThreadPoolExecutor(max_workers=concurrent_notebooks) as executor:
with concurrent.futures.ThreadPoolExecutor(
max_workers=concurrent_notebooks
) as executor:
print(f"Max workers: {executor._max_workers}")
notebook_execution_results = list(
@@ -604,9 +611,7 @@ def process_and_execute_notebooks(
else:
print(log_contents)
_save_results(results_sorted,
artifacts_bucket,
results_file)
_save_results(results_sorted, artifacts_bucket, results_file)
print("\n=== END RESULTS===\n")
.cloud-build/notebook-execution-test-cloudbuild.yaml
+1 -1
View File
@@ -36,7 +36,7 @@ steps:
- -c
- |
. workspace/env/bin/activate &&
python3 .cloud-build/execute_changed_notebooks_cli.py --test_paths_file "${_TEST_PATHS_FILE}" --base_branch "${_FORCED_BASE_BRANCH}" --container_uri ${_PYTHON_IMAGE} --staging_bucket ${_GCS_STAGING_BUCKET} --artifacts_bucket ${_GCS_STAGING_BUCKET}/executed_notebooks/PR_${_PR_NUMBER}/BUILD_${BUILD_ID} --variable_project_id ${PROJECT_ID} --variable_region ${_GCP_REGION} --variable_service_account ${_GCP_SERVICE_ACCOUNT} --variable_vpc_network "${_GPC_VPC_NETWORK_NAME}" `if [ ! -z "${_PRIVATE_POOL_NAME}" ]; then echo "--private_pool_id ${_PRIVATE_POOL_NAME}"; fi` --build_id ${BUILD_ID} --test_percent=${_TEST_PERCENT} --concurrent_notebooks=${_CONCURRENT_NOTEBOOKS}
python3 .cloud-build/execute_changed_notebooks_cli.py --test_paths_file "${_TEST_PATHS_FILE}" --base_branch "${_FORCED_BASE_BRANCH}" --container_uri ${_PYTHON_IMAGE} --staging_bucket ${_GCS_STAGING_BUCKET} --artifacts_bucket ${_GCS_STAGING_BUCKET}/executed_notebooks/PR_${_PR_NUMBER}/BUILD_${BUILD_ID} --variable_project_id ${PROJECT_ID} --variable_region ${_GCP_REGION} --variable_service_account ${_GCP_SERVICE_ACCOUNT} --variable_vpc_network "${_GCP_VPC_NETWORK_NAME}" `if [ ! -z "${_PRIVATE_POOL_NAME}" ]; then echo "--private_pool_id ${_PRIVATE_POOL_NAME}"; fi` --build_id ${BUILD_ID} --test_percent=${_TEST_PERCENT} --concurrent_notebooks=${_CONCURRENT_NOTEBOOKS}
env:
- 'IS_TESTING=1'
timeout: 86400s
.cloud-build/utils/NotebookProcessors.py
+1 -1
View File
@@ -35,7 +35,7 @@ class RemoveNoExecuteCells(Preprocessor):
class UpdateVariablesPreprocessor(Preprocessor):
def __init__(self, replacement_map: Dict):
def __init__(self, replacement_map: Dict[str, str]):
self._replacement_map = replacement_map
@staticmethod
notebooks/official/matching_engine/sdk_matching_engine_for_indexing.ipynb
+19 -51
View File
@@ -24,13 +24,12 @@
]
},
{
"attachments": {},
"cell_type": "markdown",
"metadata": {
"id": "JAPoU8Sm5E6e"
},
"source": [
"# Create Vertex AI Matching Engine index\n",
"# Create Vertex AI Vector Search index\n",
"![ ](https://www.google-analytics.com/collect?v=2&tid=G-L6X3ECH596&cid=1&en=page_view&sid=1&dt=sdk_matching_engine_for_indexing.ipynb&dl=notebooks%2Fofficial%2Fmatching_engine%2Fsdk_matching_engine_for_indexing.ipynb)\n",
"<table align=\"left\">\n",
" <td>\n",
@@ -63,7 +62,7 @@
"\n",
"This example demonstrates how to use the Vertex AI ANN Service. It is a high scale, low latency solution, to find similar vectors (or more specifically \"embeddings\") for a large corpus. Moreover, it is a fully managed offering, further reducing operational overhead. It is built upon [Approximate Nearest Neighbor (ANN) technology](https://ai.googleblog.com/2020/07/announcing-scann-efficient-vector.html) developed by Google Research.\n",
"\n",
"Learn more about [Vertex AI Matching Engine](https://cloud.google.com/vertex-ai/docs/matching-engine/overview)."
"Learn more about [Vertex AI Vector Search](https://cloud.google.com/vertex-ai/docs/matching-engine/overview)."
]
},
{
@@ -78,7 +77,7 @@
"\n",
"This tutorial uses the following Google Cloud ML services:\n",
"\n",
"- `Vertex AI Matching Engine`\n",
"- `Vertex AI Vector Search`\n",
"\n",
"The steps performed include:\n",
"\n",
@@ -110,7 +109,7 @@
"source": [
"## Installation\n",
"\n",
"Install the latest version of Cloud Storage, BigQuery and Vertex AI SDKs for Python."
"Install the latest versions of packages required to execute this notebook."
]
},
{
@@ -123,7 +122,9 @@
"source": [
"# Install the packages\n",
"! pip3 install --upgrade google-cloud-aiplatform \\\n",
" google-cloud-storage"
" google-cloud-storage \\\n",
" grpcio-tools \\\n",
" h5py"
]
},
{
@@ -353,37 +354,6 @@
" * If you run it in the colab or a Vertex AI Workbench notebook instance in a different VPC network or region, \"Create Online Queries\" section will fail."
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "i7EUnXsZhAGF"
},
"source": [
"### Installation\n",
"\n",
"Download and install the latest version of the Vertex SDK for Python."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "wyy5Lbnzg5fi"
},
"outputs": [],
"source": [
"! pip install --upgrade --quiet google-cloud-aiplatform grpcio-tools h5py"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "irSMQn6gZ19l"
},
"source": [
"Install the `h5py` to prepare sample dataset, and the `grpcio-tools` for querying against the index. "
]
},
{
"cell_type": "markdown",
"metadata": {
@@ -427,7 +397,7 @@
},
"outputs": [],
"source": [
"BUCKET_URI = \"gs://your-bucket-name-unique\" # @param {type:\"string\"}"
"BUCKET_URI = f\"gs://your-bucket-name-{PROJECT_ID}-unique\" # @param {type:\"string\"}"
]
},
{
@@ -447,7 +417,7 @@
},
"outputs": [],
"source": [
"! gsutil mb -l $REGION -p $PROJECT_ID $BUCKET_URI"
"! gsutil mb -l {REGION} -p {PROJECT_ID} {BUCKET_URI}"
]
},
{
@@ -1097,21 +1067,19 @@
},
"outputs": [],
"source": [
"import os\n",
"\n",
"delete_bucket = False\n",
"\n",
"# Force undeployment of indexes and delete endpoint\n",
"my_index_endpoint.delete(force=True)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "omj7N9iWv-Tq"
},
"outputs": [],
"source": [
"my_index_endpoint.delete(force=True)\n",
"\n",
"# Delete indexes\n",
"tree_ah_index.delete()\n",
"brute_force_index.delete()"
"brute_force_index.delete()\n",
"\n",
"if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
" ! gsutil rm -rf {BUCKET_URI}"
]
}
],