xiaohei/model_garden
1
0
Fork 0
mirror of https://github.com/GoogleCloudPlatform/vertex-ai-samples.git synced 2026-09-26 14:42:04 +00:00
Code Issues Packages Projects Releases Wiki Activity

Harden 9 (#2429)

Browse Source 
* fix: force run based on age of last run

* fix: force run based on age of last run
This commit is contained in:
Andrew Ferlitsch
2023-10-26 15:40:34 +00:00
committed by GitHub
parent 8dad9cd13f
commit 7206ea4d99
1 changed files with 21 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.cloud-build/execute_changed_notebooks_helper.py
+21 -5
View File
@@ -45,6 +45,8 @@ PYTHON_VERSION = "3.9" # Set default python version
# rolling time window for accumulating build results for selecting notebooks
MAX_RESULTS_AGE_SECONDS: int = (60 * 60) * 24 * 60 # 60 days
# maximum time since last run to force a run on the current build
MAX_AGE_BEFORE_FORCE_RUN: int = (60 * 60) * 24 * 30
def format_timedelta(delta: datetime.timedelta) -> str:
@@ -120,8 +122,8 @@ def load_results(results_bucket: str,
accumulative_results[notebook]['failed'] += build_results[notebook]['failed']
else:
accumulative_results[notebook] = build_results[notebook]
accumulative_results[notebook]['failed_last_run'] = build_results[notebook]['failed']
accumulative_results[notebook]['failed_last_date'] = time_created
accumulative_results[notebook]['failed_on_latest_run'] = build_results[notebook]['failed']
accumulative_results[notebook]['last_time_ran'] = time_created
print(accumulative_results)
except Exception as e:
@@ -140,23 +142,37 @@ def select_notebook(changed_notebook: str,
if changed_notebook in accumulative_results:
pass_count = accumulative_results[changed_notebook]['passed']
fail_count = accumulative_results[changed_notebook]['failed']
failed_on_latest_run = accumulative_results[changed_notebook]['failed_on_latest_run']
last_time_ran = accumulative_results[changed_notebook]['last_time_ran']
else:
pass_count = 1
fail_count = 0
failed_on_latest_run = 0
last_time_ran = datetime.datetime.now().replace(tzinfo=None)
# if failed on last run, select the notebook
if accumulative_results['failed_last_run']:
# If notebook has not been ran in a long time, force running it
if (datetime.datetime.now().replace(tzinfo=None) - last_time_ran).total_seconds() > MAX_AGE_BEFORE_FORCE_RUN:
should_test_do_to_age = True
else:
should_test_do_to_age = False
# if failed on the last time it was ran, select the notebook
if failed_on_latest_run:
inferred_failure_rate = 1
# otherwise, calculate the frequency of failure
else:
inferred_failure_rate = fail_count / (pass_count + fail_count)
# If failure rate is high, the chance of testing should be higher
should_test_due_to_failure = random.uniform(0, 1) <= inferred_failure_rate
#if accumulative_resultsi[changed_notebook]['latest_date_ran']
# Additionally, only test a percentage of these
should_test_due_to_random_subset = random.uniform(0, 1) <= (test_percent / 100)
if should_test_due_to_failure or should_test_due_to_random_subset:
if should_test_due_to_failure or should_test_due_to_random_subset or should_test_do_to_age:
print(f"Selected: {changed_notebook}, {should_test_due_to_failure}, {should_test_due_to_random_subset}")
return True
else: