Build/Test Explorer

github/python-bigtable Target status: Flaky

Target ran on for 1 hr, 43 min, 1 sec
5 Failed1,467 Passed1 Skipped
There is test suite information that is not visible in this view.  Switch to Tree View to see test suites.
Status
Test
Start time
Run time
Failedtest_instance_admin.test_cluster_update
Attempt 1
Attempt
target = functools.partial(<bound method PollingFuture._done_or_raise of <google.api_core.operation.Operation object at 0x7ffac00960a0>>)
predicate = <function if_exception_type.<locals>.if_exception_type_predicate at 0x7ffac24ca9d0>
sleep_generator = <generator object exponential_sleep_generator at 0x7ffac005ea50>
deadline = 60, on_error = None

def retry_target(target, predicate, sleep_generator, deadline, on_error=None):
"""Call a function and retry if it fails.

This is the lowest-level retry helper. Generally, you'll use the
higher-level retry helper :class:`Retry`.

Args:
target(Callable): The function to call and retry. This must be a
nullary function - apply arguments with `functools.partial`.
predicate (Callable[Exception]): A callable used to determine if an
exception raised by the target should be considered retryable.
It should return True to retry or False otherwise.
sleep_generator (Iterable[float]): An infinite iterator that determines
how long to sleep between retries.
deadline (float): How long to keep retrying the target. The last sleep
period is shortened as necessary, so that the last retry runs at
``deadline`` (and not considerably beyond it).
on_error (Callable[Exception]): A function to call while processing a
retryable exception. Any error raised by this function will *not*
be caught.

Returns:
Any: the return value of the target function.

Raises:
google.api_core.RetryError: If the deadline is exceeded while retrying.
ValueError: If the sleep generator stops yielding values.
Exception: If the target raises a method that isn't retryable.
"""
if deadline is not None:
deadline_datetime = datetime_helpers.utcnow() + datetime.timedelta(
seconds=deadline
)
else:
deadline_datetime = None

last_exc = None

for sleep in sleep_generator:
try:
> return target()

.nox/system-3-8/lib/python3.8/site-packages/google/api_core/retry.py:190:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

self = <google.api_core.operation.Operation object at 0x7ffac00960a0>
retry = <google.api_core.retry.Retry object at 0x7ffac24d11c0>

def _done_or_raise(self, retry=DEFAULT_RETRY):
"""Check if the future is done and raise if it's not."""
kwargs = {} if retry is DEFAULT_RETRY else {"retry": retry}

if not self.done(**kwargs):
> raise _OperationNotComplete()
E google.api_core.future.polling._OperationNotComplete

.nox/system-3-8/lib/python3.8/site-packages/google/api_core/future/polling.py:89: _OperationNotComplete

The above exception was the direct cause of the following exception:

self = <google.api_core.operation.Operation object at 0x7ffac00960a0>
timeout = 60, retry = <google.api_core.retry.Retry object at 0x7ffac24d11c0>

def _blocking_poll(self, timeout=None, retry=DEFAULT_RETRY):
"""Poll and wait for the Future to be resolved.

Args:
timeout (int):
How long (in seconds) to wait for the operation to complete.
If None, wait indefinitely.
"""
if self._result_set:
return

retry_ = self._retry.with_deadline(timeout)

try:
kwargs = {} if retry is DEFAULT_RETRY else {"retry": retry}
> retry_(self._done_or_raise)(**kwargs)

.nox/system-3-8/lib/python3.8/site-packages/google/api_core/future/polling.py:110:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

args = (), kwargs = {}
target = functools.partial(<bound method PollingFuture._done_or_raise of <google.api_core.operation.Operation object at 0x7ffac00960a0>>)
sleep_generator = <generator object exponential_sleep_generator at 0x7ffac005ea50>

@functools.wraps(func)
def retry_wrapped_func(*args, **kwargs):
"""A wrapper that calls target function with retry."""
target = functools.partial(func, *args, **kwargs)
sleep_generator = exponential_sleep_generator(
self._initial, self._maximum, multiplier=self._multiplier
)
> return retry_target(
target,
self._predicate,
sleep_generator,
self._deadline,
on_error=on_error,
)

.nox/system-3-8/lib/python3.8/site-packages/google/api_core/retry.py:283:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

target = functools.partial(<bound method PollingFuture._done_or_raise of <google.api_core.operation.Operation object at 0x7ffac00960a0>>)
predicate = <function if_exception_type.<locals>.if_exception_type_predicate at 0x7ffac24ca9d0>
sleep_generator = <generator object exponential_sleep_generator at 0x7ffac005ea50>
deadline = 60, on_error = None

def retry_target(target, predicate, sleep_generator, deadline, on_error=None):
"""Call a function and retry if it fails.

This is the lowest-level retry helper. Generally, you'll use the
higher-level retry helper :class:`Retry`.

Args:
target(Callable): The function to call and retry. This must be a
nullary function - apply arguments with `functools.partial`.
predicate (Callable[Exception]): A callable used to determine if an
exception raised by the target should be considered retryable.
It should return True to retry or False otherwise.
sleep_generator (Iterable[float]): An infinite iterator that determines
how long to sleep between retries.
deadline (float): How long to keep retrying the target. The last sleep
period is shortened as necessary, so that the last retry runs at
``deadline`` (and not considerably beyond it).
on_error (Callable[Exception]): A function to call while processing a
retryable exception. Any error raised by this function will *not*
be caught.

Returns:
Any: the return value of the target function.

Raises:
google.api_core.RetryError: If the deadline is exceeded while retrying.
ValueError: If the sleep generator stops yielding values.
Exception: If the target raises a method that isn't retryable.
"""
if deadline is not None:
deadline_datetime = datetime_helpers.utcnow() + datetime.timedelta(
seconds=deadline
)
else:
deadline_datetime = None

last_exc = None

for sleep in sleep_generator:
try:
return target()

# pylint: disable=broad-except
# This function explicitly must deal with broad exceptions.
except Exception as exc:
if not predicate(exc):
raise
last_exc = exc
if on_error is not None:
on_error(exc)

now = datetime_helpers.utcnow()

if deadline_datetime is not None:
if deadline_datetime <= now:
> raise exceptions.RetryError(
"Deadline of {:.1f}s exceeded while calling target function".format(
deadline
),
last_exc,
) from last_exc
E google.api_core.exceptions.RetryError: Deadline of 60.0s exceeded while calling target function, last exception:

.nox/system-3-8/lib/python3.8/site-packages/google/api_core/retry.py:205: RetryError

During handling of the above exception, another exception occurred:

admin_instance_populated = <google.cloud.bigtable.instance.Instance object at 0x7ffac00b3760>
admin_cluster_id = 'g-c-p-1660930396039-cluster'
admin_cluster = <google.cloud.bigtable.cluster.Cluster object at 0x7ffac096f610>
serve_nodes = 3, skip_on_emulator = None

def test_cluster_update(
admin_instance_populated,
admin_cluster_id,
admin_cluster,
serve_nodes,
skip_on_emulator,
):
new_serve_nodes = 4

admin_cluster.serve_nodes = new_serve_nodes

operation = admin_cluster.update()
operation.result(timeout=60) # Ensure the operation completes.

# Create a new cluster instance and reload it.
alt_cluster = admin_instance_populated.cluster(admin_cluster_id)
alt_cluster.reload()
assert alt_cluster.serve_nodes == new_serve_nodes

# Put the cluster back the way it was for the other test cases.
admin_cluster.serve_nodes = serve_nodes
operation = admin_cluster.update()
> operation.result(timeout=60) # Ensure the operation completes.

tests/system/test_instance_admin.py:762:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
.nox/system-3-8/lib/python3.8/site-packages/google/api_core/future/polling.py:132: in result
self._blocking_poll(timeout=timeout, **kwargs)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

self = <google.api_core.operation.Operation object at 0x7ffac00960a0>
timeout = 60, retry = <google.api_core.retry.Retry object at 0x7ffac24d11c0>

def _blocking_poll(self, timeout=None, retry=DEFAULT_RETRY):
"""Poll and wait for the Future to be resolved.

Args:
timeout (int):
How long (in seconds) to wait for the operation to complete.
If None, wait indefinitely.
"""
if self._result_set:
return

retry_ = self._retry.with_deadline(timeout)

try:
kwargs = {} if retry is DEFAULT_RETRY else {"retry": retry}
retry_(self._done_or_raise)(**kwargs)
except exceptions.RetryError:
> raise concurrent.futures.TimeoutError(
"Operation did not complete within the designated " "timeout."
)
E concurrent.futures._base.TimeoutError: Operation did not complete within the designated timeout.

.nox/system-3-8/lib/python3.8/site-packages/google/api_core/future/polling.py:112: TimeoutError
  target = functools.partial(<bound method PollingFuture._done_or_raise of <google.api_core.operation.Operation object at 0x7ffac00960a0>>)
predicate = <function if_exception_type.<locals>.if_exception_type_predicate at 0x7ffac24ca9d0>
sleep_generator = <generator object exponential_sleep_generator at 0x7ffac005ea50>
deadline = 60, on_error = None

    def retry_target(target, predicate, sleep_generator, deadline, on_error=None):
        """Call a function and retry if it fails.
    
        This is the lowest-level retry helper. Generally, you'll use the
        higher-level retry helper :class:`Retry`.
    
        Args:
            target(Callable): The function to call and retry. This must be a
                nullary function - apply arguments with `functools.partial`.
            predicate (Callable[Exception]): A callable used to determine if an
                exception raised by the target should be considered retryable.
                It should return True to retry or False otherwise.
            sleep_generator (Iterable[float]): An infinite iterator that determines
                how long to sleep between retries.
            deadline (float): How long to keep retrying the target. The last sleep
                period is shortened as necessary, so that the last retry runs at
                ``deadline`` (and not considerably beyond it).
            on_error (Callable[Exception]): A function to call while processing a
                retryable exception.  Any error raised by this function will *not*
                be caught.
    
        Returns:
            Any: the return value of the target function.
    
        Raises:
            google.api_core.RetryError: If the deadline is exceeded while retrying.
            ValueError: If the sleep generator stops yielding values.
            Exception: If the target raises a method that isn't retryable.
        """
        if deadline is not None:
            deadline_datetime = datetime_helpers.utcnow() + datetime.timedelta(
                seconds=deadline
            )
        else:
            deadline_datetime = None
    
        last_exc = None
    
        for sleep in sleep_generator:
            try:
>               return target()

.nox/system-3-8/lib/python3.8/site-packages/google/api_core/retry.py:190: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <google.api_core.operation.Operation object at 0x7ffac00960a0>
retry = <google.api_core.retry.Retry object at 0x7ffac24d11c0>

    def _done_or_raise(self, retry=DEFAULT_RETRY):
        """Check if the future is done and raise if it's not."""
        kwargs = {} if retry is DEFAULT_RETRY else {"retry": retry}
    
        if not self.done(**kwargs):
>           raise _OperationNotComplete()
E           google.api_core.future.polling._OperationNotComplete

.nox/system-3-8/lib/python3.8/site-packages/google/api_core/future/polling.py:89: _OperationNotComplete

The above exception was the direct cause of the following exception:

self = <google.api_core.operation.Operation object at 0x7ffac00960a0>
timeout = 60, retry = <google.api_core.retry.Retry object at 0x7ffac24d11c0>

    def _blocking_poll(self, timeout=None, retry=DEFAULT_RETRY):
        """Poll and wait for the Future to be resolved.
    
        Args:
            timeout (int):
                How long (in seconds) to wait for the operation to complete.
                If None, wait indefinitely.
        """
        if self._result_set:
            return
    
        retry_ = self._retry.with_deadline(timeout)
    
        try:
            kwargs = {} if retry is DEFAULT_RETRY else {"retry": retry}
>           retry_(self._done_or_raise)(**kwargs)

.nox/system-3-8/lib/python3.8/site-packages/google/api_core/future/polling.py:110: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

args = (), kwargs = {}
target = functools.partial(<bound method PollingFuture._done_or_raise of <google.api_core.operation.Operation object at 0x7ffac00960a0>>)
sleep_generator = <generator object exponential_sleep_generator at 0x7ffac005ea50>

    @functools.wraps(func)
    def retry_wrapped_func(*args, **kwargs):
        """A wrapper that calls target function with retry."""
        target = functools.partial(func, *args, **kwargs)
        sleep_generator = exponential_sleep_generator(
            self._initial, self._maximum, multiplier=self._multiplier
        )
>       return retry_target(
            target,
            self._predicate,
            sleep_generator,
            self._deadline,
            on_error=on_error,
        )

.nox/system-3-8/lib/python3.8/site-packages/google/api_core/retry.py:283: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

target = functools.partial(<bound method PollingFuture._done_or_raise of <google.api_core.operation.Operation object at 0x7ffac00960a0>>)
predicate = <function if_exception_type.<locals>.if_exception_type_predicate at 0x7ffac24ca9d0>
sleep_generator = <generator object exponential_sleep_generator at 0x7ffac005ea50>
deadline = 60, on_error = None

    def retry_target(target, predicate, sleep_generator, deadline, on_error=None):
        """Call a function and retry if it fails.
    
        This is the lowest-level retry helper. Generally, you'll use the
        higher-level retry helper :class:`Retry`.
    
        Args:
            target(Callable): The function to call and retry. This must be a
                nullary function - apply arguments with `functools.partial`.
            predicate (Callable[Exception]): A callable used to determine if an
                exception raised by the target should be considered retryable.
                It should return True to retry or False otherwise.
            sleep_generator (Iterable[float]): An infinite iterator that determines
                how long to sleep between retries.
            deadline (float): How long to keep retrying the target. The last sleep
                period is shortened as necessary, so that the last retry runs at
                ``deadline`` (and not considerably beyond it).
            on_error (Callable[Exception]): A function to call while processing a
                retryable exception.  Any error raised by this function will *not*
                be caught.
    
        Returns:
            Any: the return value of the target function.
    
        Raises:
            google.api_core.RetryError: If the deadline is exceeded while retrying.
            ValueError: If the sleep generator stops yielding values.
            Exception: If the target raises a method that isn't retryable.
        """
        if deadline is not None:
            deadline_datetime = datetime_helpers.utcnow() + datetime.timedelta(
                seconds=deadline
            )
        else:
            deadline_datetime = None
    
        last_exc = None
    
        for sleep in sleep_generator:
            try:
                return target()
    
            # pylint: disable=broad-except
            # This function explicitly must deal with broad exceptions.
            except Exception as exc:
                if not predicate(exc):
                    raise
                last_exc = exc
                if on_error is not None:
                    on_error(exc)
    
            now = datetime_helpers.utcnow()
    
            if deadline_datetime is not None:
                if deadline_datetime <= now:
>                   raise exceptions.RetryError(
                        "Deadline of {:.1f}s exceeded while calling target function".format(
                            deadline
                        ),
                        last_exc,
                    ) from last_exc
E                   google.api_core.exceptions.RetryError: Deadline of 60.0s exceeded while calling target function, last exception:

.nox/system-3-8/lib/python3.8/site-packages/google/api_core/retry.py:205: RetryError

During handling of the above exception, another exception occurred:

admin_instance_populated = <google.cloud.bigtable.instance.Instance object at 0x7ffac00b3760>
admin_cluster_id = 'g-c-p-1660930396039-cluster'
admin_cluster = <google.cloud.bigtable.cluster.Cluster object at 0x7ffac096f610>
serve_nodes = 3, skip_on_emulator = None

    def test_cluster_update(
        admin_instance_populated,
        admin_cluster_id,
        admin_cluster,
        serve_nodes,
        skip_on_emulator,
    ):
        new_serve_nodes = 4
    
        admin_cluster.serve_nodes = new_serve_nodes
    
        operation = admin_cluster.update()
        operation.result(timeout=60)  # Ensure the operation completes.
    
        # Create a new cluster instance and reload it.
        alt_cluster = admin_instance_populated.cluster(admin_cluster_id)
        alt_cluster.reload()
        assert alt_cluster.serve_nodes == new_serve_nodes
    
        # Put the cluster back the way it was for the other test cases.
        admin_cluster.serve_nodes = serve_nodes
        operation = admin_cluster.update()
>       operation.result(timeout=60)  # Ensure the operation completes.

tests/system/test_instance_admin.py:762: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
.nox/system-3-8/lib/python3.8/site-packages/google/api_core/future/polling.py:132: in result
    self._blocking_poll(timeout=timeout, **kwargs)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <google.api_core.operation.Operation object at 0x7ffac00960a0>
timeout = 60, retry = <google.api_core.retry.Retry object at 0x7ffac24d11c0>

    def _blocking_poll(self, timeout=None, retry=DEFAULT_RETRY):
        """Poll and wait for the Future to be resolved.
    
        Args:
            timeout (int):
                How long (in seconds) to wait for the operation to complete.
                If None, wait indefinitely.
        """
        if self._result_set:
            return
    
        retry_ = self._retry.with_deadline(timeout)
    
        try:
            kwargs = {} if retry is DEFAULT_RETRY else {"retry": retry}
            retry_(self._done_or_raise)(**kwargs)
        except exceptions.RetryError:
>           raise concurrent.futures.TimeoutError(
                "Operation did not complete within the designated " "timeout."
            )
E           concurrent.futures._base.TimeoutError: Operation did not complete within the designated timeout.

.nox/system-3-8/lib/python3.8/site-packages/google/api_core/future/polling.py:112: TimeoutError
Items per page:
1 – 5 of 5