fix(kubernetes): temporary solution for updated k8s python client

kubernetes/base/watch/__init__.py

@@ -0,0 +1,15 @@
+# Copyright 2016 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from .watch import Watch

kubernetes/base/watch/watch.py

@@ -0,0 +1,213 @@
+# Copyright 2016 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import pydoc
+import sys
+
+from kubernetes import client
+
+PYDOC_RETURN_LABEL = ":return:"
+PYDOC_FOLLOW_PARAM = ":param bool follow:"
+
+# Removing this suffix from return type name should give us event's object
+# type. e.g., if list_namespaces() returns "NamespaceList" type,
+# then list_namespaces(watch=true) returns a stream of events with objects
+# of type "Namespace". In case this assumption is not true, user should
+# provide return_type to Watch class's __init__.
+TYPE_LIST_SUFFIX = "List"
+
+
+PY2 = sys.version_info[0] == 2
+if PY2:
+    import httplib
+    HTTP_STATUS_GONE = httplib.GONE
+else:
+    import http
+    HTTP_STATUS_GONE = http.HTTPStatus.GONE
+
+
+class SimpleNamespace:
+
+    def __init__(self, **kwargs):
+        self.__dict__.update(kwargs)
+
+
+def _find_return_type(func):
+    for line in pydoc.getdoc(func).splitlines():
+        if line.startswith(PYDOC_RETURN_LABEL):
+            return line[len(PYDOC_RETURN_LABEL):].strip()
+    return ""
+
+
+def iter_resp_lines(resp):
+    buffer = bytearray()
+    for segment in resp.stream(amt=None, decode_content=False):
+
+        # Append the segment (chunk) to the buffer
+        #
+        # Performance note: depending on contents of buffer and the type+value of segment,
+        # encoding segment into the buffer could be a wasteful step. The approach used here
+        # simplifies the logic farther down, but in the future it may be reasonable to
+        # sacrifice readability for performance.
+        if isinstance(segment, bytes):
+            buffer.extend(segment)
+        elif isinstance(segment, str):
+            buffer.extend(segment.encode("utf-8"))
+        else:
+            raise TypeError(
+                f"Received invalid segment type, {type(segment)}, from stream. Accepts only 'str' or 'bytes'.")
+
+        # Split by newline (safe for utf-8 because multi-byte sequences cannot contain the newline byte)
+        next_newline = buffer.find(b'\n')
+        while next_newline != -1:
+            # Convert bytes to a valid utf-8 string, replacing any invalid utf-8 with the '<27>' character
+            line = buffer[:next_newline].decode(
+                "utf-8", errors="replace")
+            buffer = buffer[next_newline+1:]
+            if line:
+                yield line
+            next_newline = buffer.find(b'\n')
+
+
+class Watch(object):
+
+    def __init__(self, return_type=None):
+        self._raw_return_type = return_type
+        self._stop = False
+        self._api_client = client.ApiClient()
+        self.resource_version = None
+
+    def stop(self):
+        self._stop = True
+
+    def get_return_type(self, func):
+        if self._raw_return_type:
+            return self._raw_return_type
+        return_type = _find_return_type(func)
+        if return_type.endswith(TYPE_LIST_SUFFIX):
+            return return_type[:-len(TYPE_LIST_SUFFIX)]
+        return return_type
+
+    def get_watch_argument_name(self, func):
+        if PYDOC_FOLLOW_PARAM in pydoc.getdoc(func):
+            return 'follow'
+        else:
+            return 'watch'
+
+    def unmarshal_event(self, data, return_type):
+        js = json.loads(data)
+        js['raw_object'] = js['object']
+        # BOOKMARK event is treated the same as ERROR for a quick fix of
+        # decoding exception
+        # TODO: make use of the resource_version in BOOKMARK event for more
+        # efficient WATCH
+        if return_type and js['type'] != 'ERROR' and js['type'] != 'BOOKMARK':
+            obj = SimpleNamespace(data=json.dumps(js['raw_object']))
+            js['object'] = self._api_client.deserialize(obj, return_type)
+            if hasattr(js['object'], 'metadata'):
+                self.resource_version = js['object'].metadata.resource_version
+            # For custom objects that we don't have model defined, json
+            # deserialization results in dictionary
+            elif (isinstance(js['object'], dict) and 'metadata' in js['object']
+                  and 'resourceVersion' in js['object']['metadata']):
+                self.resource_version = js['object']['metadata'][
+                    'resourceVersion']
+        return js
+
+    def stream(self, func, *args, **kwargs):
+        """Watch an API resource and stream the result back via a generator.
+
+        Note that watching an API resource can expire. The method tries to
+        resume automatically once from the last result, but if that last result
+        is too old as well, an `ApiException` exception will be thrown with
+        ``code`` 410. In that case you have to recover yourself, probably
+        by listing the API resource to obtain the latest state and then
+        watching from that state on by setting ``resource_version`` to
+        one returned from listing.
+
+        :param func: The API function pointer. Any parameter to the function
+                     can be passed after this parameter.
+
+        :return: Event object with these keys:
+                   'type': The type of event such as "ADDED", "DELETED", etc.
+                   'raw_object': a dict representing the watched object.
+                   'object': A model representation of raw_object. The name of
+                             model will be determined based on
+                             the func's doc string. If it cannot be determined,
+                             'object' value will be the same as 'raw_object'.
+
+        Example:
+            v1 = kubernetes.client.CoreV1Api()
+            watch = kubernetes.watch.Watch()
+            for e in watch.stream(v1.list_namespace, resource_version=1127):
+                type_ = e['type']
+                object_ = e['object']  # object is one of type return_type
+                raw_object = e['raw_object']  # raw_object is a dict
+                ...
+                if should_stop:
+                    watch.stop()
+        """
+
+        self._stop = False
+        return_type = self.get_return_type(func)
+        watch_arg = self.get_watch_argument_name(func)
+        kwargs[watch_arg] = True
+        kwargs['_preload_content'] = False
+        if 'resource_version' in kwargs:
+            self.resource_version = kwargs['resource_version']
+
+        # Do not attempt retries if user specifies a timeout.
+        # We want to ensure we are returning within that timeout.
+        disable_retries = ('timeout_seconds' in kwargs)
+        retry_after_410 = False
+        while True:
+            resp = func(*args, **kwargs)
+            try:
+                for line in iter_resp_lines(resp):
+                    # unmarshal when we are receiving events from watch,
+                    # return raw string when we are streaming log
+                    if watch_arg == "watch":
+                        event = self.unmarshal_event(line, return_type)
+                        if isinstance(event, dict) \
+                                and event['type'] == 'ERROR':
+                            obj = event['raw_object']
+                            # Current request expired, let's retry, (if enabled)
+                            # but only if we have not already retried.
+                            if not disable_retries and not retry_after_410 and \
+                                    obj['code'] == HTTP_STATUS_GONE:
+                                retry_after_410 = True
+                                break
+                            else:
+                                reason = "%s: %s" % (
+                                    obj['reason'], obj['message'])
+                                raise client.rest.ApiException(
+                                    status=obj['code'], reason=reason)
+                        else:
+                            retry_after_410 = False
+                            yield event
+                    else:
+                        yield line
+                    if self._stop:
+                        break
+            finally:
+                resp.close()
+                resp.release_conn()
+                if self.resource_version is not None:
+                    kwargs['resource_version'] = self.resource_version
+                else:
+                    self._stop = True
+
+            if self._stop or disable_retries:
+                break

kubernetes/base/watch/watch_test.py

@@ -0,0 +1,494 @@
+# Copyright 2016 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+
+from unittest.mock import Mock, call
+
+from kubernetes import client
+
+from .watch import Watch
+
+
+class WatchTests(unittest.TestCase):
+    def setUp(self):
+        # counter for a test that needs test global state
+        self.callcount = 0
+
+    def test_watch_with_decode(self):
+        fake_resp = Mock()
+        fake_resp.close = Mock()
+        fake_resp.release_conn = Mock()
+        fake_resp.stream = Mock(
+            return_value=[
+                '{"type": "ADDED", "object": {"metadata": {"name": "test1",'
+                '"resourceVersion": "1"}, "spec": {}, "status": {}}}\n',
+                '{"type": "ADDED", "object": {"metadata": {"name": "test2",'
+                '"resourceVersion": "2"}, "spec": {}, "sta',
+                'tus": {}}}\n'
+                '{"type": "ADDED", "object": {"metadata": {"name": "test3",'
+                '"resourceVersion": "3"}, "spec": {}, "status": {}}}\n',
+                'should_not_happened\n'])
+
+        fake_api = Mock()
+        fake_api.get_namespaces = Mock(return_value=fake_resp)
+        fake_api.get_namespaces.__doc__ = ':return: V1NamespaceList'
+
+        w = Watch()
+        count = 1
+        for e in w.stream(fake_api.get_namespaces):
+            self.assertEqual("ADDED", e['type'])
+            # make sure decoder worked and we got a model with the right name
+            self.assertEqual("test%d" % count, e['object'].metadata.name)
+            # make sure decoder worked and updated Watch.resource_version
+            self.assertEqual(
+                "%d" % count, e['object'].metadata.resource_version)
+            self.assertEqual("%d" % count, w.resource_version)
+            count += 1
+            # make sure we can stop the watch and the last event with won't be
+            # returned
+            if count == 4:
+                w.stop()
+
+        # make sure that all three records were consumed by the stream
+        self.assertEqual(4, count)
+
+        fake_api.get_namespaces.assert_called_once_with(
+            _preload_content=False, watch=True)
+        fake_resp.stream.assert_called_once_with(
+            amt=None, decode_content=False)
+        fake_resp.close.assert_called_once()
+        fake_resp.release_conn.assert_called_once()
+
+    def test_watch_with_interspersed_newlines(self):
+        fake_resp = Mock()
+        fake_resp.close = Mock()
+        fake_resp.release_conn = Mock()
+        fake_resp.stream = Mock(
+            return_value=[
+                '\n',
+                '{"type": "ADDED", "object": {"metadata":',
+                '{"name": "test1","resourceVersion": "1"}}}\n{"type": "ADDED", ',
+                '"object": {"metadata": {"name": "test2", "resourceVersion": "2"}}}\n',
+                '\n',
+                '',
+                '{"type": "ADDED", "object": {"metadata": {"name": "test3", "resourceVersion": "3"}}}\n',
+                '\n\n\n',
+                '\n',
+            ])
+
+        fake_api = Mock()
+        fake_api.get_namespaces = Mock(return_value=fake_resp)
+        fake_api.get_namespaces.__doc__ = ':return: V1NamespaceList'
+
+        w = Watch()
+        count = 0
+
+        # Consume all test events from the mock service, stopping when no more data is available.
+        # Note that "timeout_seconds" below is not a timeout; rather, it disables retries and is
+        # the only way to do so. Without that, the stream will re-read the test data forever.
+        for e in w.stream(fake_api.get_namespaces, timeout_seconds=1):
+            count += 1
+            self.assertEqual("test%d" % count, e['object'].metadata.name)
+        self.assertEqual(3, count)
+
+    def test_watch_with_multibyte_utf8(self):
+        fake_resp = Mock()
+        fake_resp.close = Mock()
+        fake_resp.release_conn = Mock()
+        fake_resp.stream = Mock(
+            return_value=[
+                # two-byte utf-8 character
+                '{"type":"MODIFIED","object":{"data":{"utf-8":"© 1"},"metadata":{"name":"test1","resourceVersion":"1"}}}\n',
+                # same copyright character expressed as bytes
+                b'{"type":"MODIFIED","object":{"data":{"utf-8":"\xC2\xA9 2"},"metadata":{"name":"test2","resourceVersion":"2"}}}\n'
+                # same copyright character with bytes split across two stream chunks
+                b'{"type":"MODIFIED","object":{"data":{"utf-8":"\xC2',
+                b'\xA9 3"},"metadata":{"n',
+                # more chunks of the same event, sent as a mix of bytes and strings
+                'ame":"test3","resourceVersion":"3"',
+                '}}}',
+                b'\n'
+            ])
+
+        fake_api = Mock()
+        fake_api.get_configmaps = Mock(return_value=fake_resp)
+        fake_api.get_configmaps.__doc__ = ':return: V1ConfigMapList'
+
+        w = Watch()
+        count = 0
+
+        # Consume all test events from the mock service, stopping when no more data is available.
+        # Note that "timeout_seconds" below is not a timeout; rather, it disables retries and is
+        # the only way to do so. Without that, the stream will re-read the test data forever.
+        for event in w.stream(fake_api.get_configmaps, timeout_seconds=1):
+            count += 1
+            self.assertEqual("MODIFIED", event['type'])
+            self.assertEqual("test%d" % count, event['object'].metadata.name)
+            self.assertEqual("© %d" % count, event['object'].data["utf-8"])
+            self.assertEqual(
+                "%d" % count, event['object'].metadata.resource_version)
+            self.assertEqual("%d" % count, w.resource_version)
+        self.assertEqual(3, count)
+
+    def test_watch_with_invalid_utf8(self):
+        fake_resp = Mock()
+        fake_resp.close = Mock()
+        fake_resp.release_conn = Mock()
+        fake_resp.stream = Mock(
+            # test 1 uses 1 invalid utf-8 byte
+            # test 2 uses a sequence of 2 invalid utf-8 bytes
+            # test 3 uses a sequence of 3 invalid utf-8 bytes
+            return_value=[
+                # utf-8 sequence for 😄 is \xF0\x9F\x98\x84
+                # all other sequences below are invalid
+                # ref: https://www.w3.org/2001/06/utf-8-wrong/UTF-8-test.html
+                b'{"type":"MODIFIED","object":{"data":{"utf-8":"\xF0\x9F\x98\x84 1","invalid":"\x80 1"},"metadata":{"name":"test1"}}}\n',
+                b'{"type":"MODIFIED","object":{"data":{"utf-8":"\xF0\x9F\x98\x84 2","invalid":"\xC0\xAF 2"},"metadata":{"name":"test2"}}}\n',
+                # mix bytes/strings and split byte sequences across chunks
+                b'{"type":"MODIFIED","object":{"data":{"utf-8":"\xF0\x9F\x98',
+                b'\x84 ',
+                b'',
+                b'3","invalid":"\xE0\x80',
+                b'\xAF ',
+                '3"},"metadata":{"n',
+                'ame":"test3"',
+                '}}}',
+                b'\n'
+            ])
+
+        fake_api = Mock()
+        fake_api.get_configmaps = Mock(return_value=fake_resp)
+        fake_api.get_configmaps.__doc__ = ':return: V1ConfigMapList'
+
+        w = Watch()
+        count = 0
+
+        # Consume all test events from the mock service, stopping when no more data is available.
+        # Note that "timeout_seconds" below is not a timeout; rather, it disables retries and is
+        # the only way to do so. Without that, the stream will re-read the test data forever.
+        for event in w.stream(fake_api.get_configmaps, timeout_seconds=1):
+            count += 1
+            self.assertEqual("MODIFIED", event['type'])
+            self.assertEqual("test%d" % count, event['object'].metadata.name)
+            self.assertEqual("😄 %d" % count, event['object'].data["utf-8"])
+            # expect N replacement characters in test N
+            self.assertEqual("<EFBFBD> %d".replace('<EFBFBD>', '<EFBFBD>'*count) %
+                             count, event['object'].data["invalid"])
+        self.assertEqual(3, count)
+
+    def test_watch_for_follow(self):
+        fake_resp = Mock()
+        fake_resp.close = Mock()
+        fake_resp.release_conn = Mock()
+        fake_resp.stream = Mock(
+            return_value=[
+                'log_line_1\n',
+                'log_line_2\n'])
+
+        fake_api = Mock()
+        fake_api.read_namespaced_pod_log = Mock(return_value=fake_resp)
+        fake_api.read_namespaced_pod_log.__doc__ = ':param bool follow:\n:return: str'
+
+        w = Watch()
+        count = 1
+        for e in w.stream(fake_api.read_namespaced_pod_log):
+            self.assertEqual("log_line_1", e)
+            count += 1
+            # make sure we can stop the watch and the last event with won't be
+            # returned
+            if count == 2:
+                w.stop()
+
+        fake_api.read_namespaced_pod_log.assert_called_once_with(
+            _preload_content=False, follow=True)
+        fake_resp.stream.assert_called_once_with(
+            amt=None, decode_content=False)
+        fake_resp.close.assert_called_once()
+        fake_resp.release_conn.assert_called_once()
+
+    def test_watch_resource_version_set(self):
+        # https://github.com/kubernetes-client/python/issues/700
+        # ensure watching from a resource version does reset to resource
+        # version 0 after k8s resets the watch connection
+        fake_resp = Mock()
+        fake_resp.close = Mock()
+        fake_resp.release_conn = Mock()
+        values = [
+            '{"type": "ADDED", "object": {"metadata": {"name": "test1",'
+            '"resourceVersion": "1"}, "spec": {}, "status": {}}}\n',
+            '{"type": "ADDED", "object": {"metadata": {"name": "test2",'
+            '"resourceVersion": "2"}, "spec": {}, "sta',
+            'tus": {}}}\n'
+            '{"type": "ADDED", "object": {"metadata": {"name": "test3",'
+            '"resourceVersion": "3"}, "spec": {}, "status": {}}}\n'
+        ]
+
+        # return nothing on the first call and values on the second
+        # this emulates a watch from a rv that returns nothing in the first k8s
+        # watch reset and values later
+
+        def get_values(*args, **kwargs):
+            self.callcount += 1
+            if self.callcount == 1:
+                return []
+            else:
+                return values
+
+        fake_resp.stream = Mock(
+            side_effect=get_values)
+
+        fake_api = Mock()
+        fake_api.get_namespaces = Mock(return_value=fake_resp)
+        fake_api.get_namespaces.__doc__ = ':return: V1NamespaceList'
+
+        w = Watch()
+        # ensure we keep our requested resource version or the version latest
+        # returned version when the existing versions are older than the
+        # requested version
+        # needed for the list existing objects, then watch from there use case
+        calls = []
+
+        iterations = 2
+        # first two calls must use the passed rv, the first call is a
+        # "reset" and does not actually return anything
+        # the second call must use the same rv but will return values
+        # (with a wrong rv but a real cluster would behave correctly)
+        # calls following that will use the rv from those returned values
+        calls.append(call(_preload_content=False, watch=True,
+                          resource_version="5"))
+        calls.append(call(_preload_content=False, watch=True,
+                          resource_version="5"))
+        for i in range(iterations):
+            # ideally we want 5 here but as rv must be treated as an
+            # opaque value we cannot interpret it and order it so rely
+            # on k8s returning the events completely and in order
+            calls.append(call(_preload_content=False, watch=True,
+                              resource_version="3"))
+
+        for c, e in enumerate(w.stream(fake_api.get_namespaces,
+                                       resource_version="5")):
+            if c == len(values) * iterations:
+                w.stop()
+
+        # check calls are in the list, gives good error output
+        fake_api.get_namespaces.assert_has_calls(calls)
+        # more strict test with worse error message
+        self.assertEqual(fake_api.get_namespaces.mock_calls, calls)
+
+    def test_watch_stream_twice(self):
+        w = Watch(float)
+        for step in ['first', 'second']:
+            fake_resp = Mock()
+            fake_resp.close = Mock()
+            fake_resp.release_conn = Mock()
+            fake_resp.stream = Mock(
+                return_value=['{"type": "ADDED", "object": 1}\n'] * 4)
+
+            fake_api = Mock()
+            fake_api.get_namespaces = Mock(return_value=fake_resp)
+            fake_api.get_namespaces.__doc__ = ':return: V1NamespaceList'
+
+            count = 1
+            for e in w.stream(fake_api.get_namespaces):
+                count += 1
+                if count == 3:
+                    w.stop()
+
+            self.assertEqual(count, 3)
+            fake_api.get_namespaces.assert_called_once_with(
+                _preload_content=False, watch=True)
+            fake_resp.stream.assert_called_once_with(
+                amt=None, decode_content=False)
+            fake_resp.close.assert_called_once()
+            fake_resp.release_conn.assert_called_once()
+
+    def test_watch_stream_loop(self):
+        w = Watch(float)
+
+        fake_resp = Mock()
+        fake_resp.close = Mock()
+        fake_resp.release_conn = Mock()
+        fake_resp.stream = Mock(
+            return_value=['{"type": "ADDED", "object": 1}\n'])
+
+        fake_api = Mock()
+        fake_api.get_namespaces = Mock(return_value=fake_resp)
+        fake_api.get_namespaces.__doc__ = ':return: V1NamespaceList'
+
+        count = 0
+
+        # when timeout_seconds is set, auto-exist when timeout reaches
+        for e in w.stream(fake_api.get_namespaces, timeout_seconds=1):
+            count = count + 1
+        self.assertEqual(count, 1)
+
+        # when no timeout_seconds, only exist when w.stop() is called
+        for e in w.stream(fake_api.get_namespaces):
+            count = count + 1
+            if count == 2:
+                w.stop()
+
+        self.assertEqual(count, 2)
+        self.assertEqual(fake_api.get_namespaces.call_count, 2)
+        self.assertEqual(fake_resp.stream.call_count, 2)
+        self.assertEqual(fake_resp.close.call_count, 2)
+        self.assertEqual(fake_resp.release_conn.call_count, 2)
+
+    def test_unmarshal_with_float_object(self):
+        w = Watch()
+        event = w.unmarshal_event('{"type": "ADDED", "object": 1}', 'float')
+        self.assertEqual("ADDED", event['type'])
+        self.assertEqual(1.0, event['object'])
+        self.assertTrue(isinstance(event['object'], float))
+        self.assertEqual(1, event['raw_object'])
+
+    def test_unmarshal_with_no_return_type(self):
+        w = Watch()
+        event = w.unmarshal_event('{"type": "ADDED", "object": ["test1"]}',
+                                  None)
+        self.assertEqual("ADDED", event['type'])
+        self.assertEqual(["test1"], event['object'])
+        self.assertEqual(["test1"], event['raw_object'])
+
+    def test_unmarshal_with_custom_object(self):
+        w = Watch()
+        event = w.unmarshal_event('{"type": "ADDED", "object": {"apiVersion":'
+                                  '"test.com/v1beta1","kind":"foo","metadata":'
+                                  '{"name": "bar", "resourceVersion": "1"}}}',
+                                  'object')
+        self.assertEqual("ADDED", event['type'])
+        # make sure decoder deserialized json into dictionary and updated
+        # Watch.resource_version
+        self.assertTrue(isinstance(event['object'], dict))
+        self.assertEqual("1", event['object']['metadata']['resourceVersion'])
+        self.assertEqual("1", w.resource_version)
+
+    def test_unmarshal_with_bookmark(self):
+        w = Watch()
+        event = w.unmarshal_event(
+            '{"type":"BOOKMARK","object":{"kind":"Job","apiVersion":"batch/v1"'
+            ',"metadata":{"resourceVersion":"1"},"spec":{"template":{'
+            '"metadata":{},"spec":{"containers":null}}},"status":{}}}',
+            'V1Job')
+        self.assertEqual("BOOKMARK", event['type'])
+        # Watch.resource_version is *not* updated, as BOOKMARK is treated the
+        # same as ERROR for a quick fix of decoding exception,
+        # resource_version in BOOKMARK is *not* used at all.
+        self.assertEqual(None, w.resource_version)
+
+    def test_watch_with_exception(self):
+        fake_resp = Mock()
+        fake_resp.close = Mock()
+        fake_resp.release_conn = Mock()
+        fake_resp.stream = Mock(side_effect=KeyError('expected'))
+
+        fake_api = Mock()
+        fake_api.get_thing = Mock(return_value=fake_resp)
+
+        w = Watch()
+        try:
+            for _ in w.stream(fake_api.get_thing):
+                self.fail(self, "Should fail on exception.")
+        except KeyError:
+            pass
+            # expected
+
+        fake_api.get_thing.assert_called_once_with(
+            _preload_content=False, watch=True)
+        fake_resp.stream.assert_called_once_with(
+            amt=None, decode_content=False)
+        fake_resp.close.assert_called_once()
+        fake_resp.release_conn.assert_called_once()
+
+    def test_watch_with_error_event(self):
+        fake_resp = Mock()
+        fake_resp.close = Mock()
+        fake_resp.release_conn = Mock()
+        fake_resp.stream = Mock(
+            return_value=[
+                '{"type": "ERROR", "object": {"code": 410, '
+                '"reason": "Gone", "message": "error message"}}\n'])
+
+        fake_api = Mock()
+        fake_api.get_thing = Mock(return_value=fake_resp)
+
+        w = Watch()
+        # No events are generated when no initial resourceVersion is passed
+        # No retry is attempted either, preventing an ApiException
+        assert not list(w.stream(fake_api.get_thing))
+
+        fake_api.get_thing.assert_called_once_with(
+            _preload_content=False, watch=True)
+        fake_resp.stream.assert_called_once_with(
+            amt=None, decode_content=False)
+        fake_resp.close.assert_called_once()
+        fake_resp.release_conn.assert_called_once()
+
+    def test_watch_retries_on_error_event(self):
+        fake_resp = Mock()
+        fake_resp.close = Mock()
+        fake_resp.release_conn = Mock()
+        fake_resp.stream = Mock(
+            return_value=[
+                '{"type": "ERROR", "object": {"code": 410, '
+                '"reason": "Gone", "message": "error message"}}\n'])
+
+        fake_api = Mock()
+        fake_api.get_thing = Mock(return_value=fake_resp)
+
+        w = Watch()
+        try:
+            for _ in w.stream(fake_api.get_thing, resource_version=0):
+                self.fail(self, "Should fail with ApiException.")
+        except client.rest.ApiException:
+            pass
+
+        # Two calls should be expected during a retry
+        fake_api.get_thing.assert_has_calls(
+            [call(resource_version=0, _preload_content=False, watch=True)] * 2)
+        fake_resp.stream.assert_has_calls(
+            [call(amt=None, decode_content=False)] * 2)
+        assert fake_resp.close.call_count == 2
+        assert fake_resp.release_conn.call_count == 2
+
+    def test_watch_with_error_event_and_timeout_param(self):
+        fake_resp = Mock()
+        fake_resp.close = Mock()
+        fake_resp.release_conn = Mock()
+        fake_resp.stream = Mock(
+            return_value=[
+                '{"type": "ERROR", "object": {"code": 410, '
+                '"reason": "Gone", "message": "error message"}}\n'])
+
+        fake_api = Mock()
+        fake_api.get_thing = Mock(return_value=fake_resp)
+
+        w = Watch()
+        try:
+            for _ in w.stream(fake_api.get_thing, timeout_seconds=10):
+                self.fail(self, "Should fail with ApiException.")
+        except client.rest.ApiException:
+            pass
+
+        fake_api.get_thing.assert_called_once_with(
+            _preload_content=False, watch=True, timeout_seconds=10)
+        fake_resp.stream.assert_called_once_with(
+            amt=None, decode_content=False)
+        fake_resp.close.assert_called_once()
+        fake_resp.release_conn.assert_called_once()
+
+
+if __name__ == '__main__':
+    unittest.main()