Compare commits

...
Author SHA1 Message Date
Andrew Ferlitsch 8106bbe93b fix: missing install 2023-04-11 23:11:42 +00:00
Andrew Ferlitsch e71e4aa5b9 update: thread safe limiter 2023-04-11 22:57:20 +00:00
Andrew Ferlitsch 9ec528c07f update: thread safe limiter 2023-04-11 22:11:09 +00:00
2 changed files with 8 additions and 6 deletions
@@ -32,7 +32,8 @@ import execute_notebook_helper
import execute_notebook_remote
import nbformat
from google.cloud.devtools.cloudbuild_v1.types import BuildOperationMetadata
from ratemate import RateLimit
from pyrate_limiter import (Duration, RequestRate,
Limiter, FileLockSQLiteBucket)
from tabulate import tabulate
from utils import NotebookProcessors, util
@@ -156,9 +157,10 @@ def _create_tag(filepath: str) -> str:
return tag
rate_limit = RateLimit(max_count=10, per=60, greedy=True)
rate_limit = RequestRate(10, Duration.HOUR)
limiter = Limiter(rate_limit, bucket_class=FileLockSQLiteBucket)
@limiter.ratelimit('testing', delay=True, max_delay=360)
def process_and_execute_notebook(
container_uri: str,
staging_bucket: str,
@@ -172,7 +174,6 @@ def process_and_execute_notebook(
notebook: str,
should_get_tail_logs: bool = False,
) -> NotebookExecutionResult:
rate_limit.wait() # wait before creating the task
print(f"Running notebook: {notebook}")
+3 -2
View File
@@ -9,6 +9,7 @@ tabulate
google-cloud-aiplatform
google-cloud-storage
google-cloud-build
ratemate
GitPython
tqdm
tqdm
pyrate-limiter
filelock