From 6a8dc6da1031cb9011e859a9347b33f7d79baef0 Mon Sep 17 00:00:00 2001 From: blag Date: Sun, 29 Nov 2020 02:13:43 -0800 Subject: [PATCH 01/35] Refactor FileWatchSensor to use watchdog instead of our logshipper fork --- contrib/linux/sensors/file_watch_sensor.py | 235 ++++++++++++++++++--- 1 file changed, 203 insertions(+), 32 deletions(-) diff --git a/contrib/linux/sensors/file_watch_sensor.py b/contrib/linux/sensors/file_watch_sensor.py index 2597d63926..a4e58ae0e5 100644 --- a/contrib/linux/sensors/file_watch_sensor.py +++ b/contrib/linux/sensors/file_watch_sensor.py @@ -14,38 +14,202 @@ # limitations under the License. import os +import signal +import time +import sys -import eventlet +from watchdog.observers import Observer +from watchdog.events import FileSystemEventHandler -from logshipper.tail import Tail +try: + from st2reactor.sensor.base import Sensor +except ImportError: + Sensor = object -from st2reactor.sensor.base import Sensor + +class FileEventHandler(FileSystemEventHandler): + def __init__(self, *args, callbacks=None, **kwargs): + self.callbacks = callbacks or {} + + def dispatch(self, event): + if not event.is_synthetic and not event.is_directory: + super().dispatch(event) + + def on_created(self, event): + cb = self.callbacks.get('created') + if cb: + cb(event=event) + + def on_modified(self, event): + cb = self.callbacks.get('modified') + if cb: + cb(event=event) + + def on_moved(self, event): + cb = self.callbacks.get('moved') + if cb: + cb(event=event) + + def on_deleted(self, event): + cb = self.callbacks.get('deleted') + if cb: + cb(event=event) + + +class SingleFileTail(object): + def __init__(self, path, handler, read_all=False, observer=None): + self.path = path + self.handler = handler + self.read_all = read_all + self.buffer = '' + self.observer = observer or Observer() + + self.open() + + def read(self, event=None): + while True: + # Buffer 1024 bytes at a time + buff = os.read(self.fd, 1024) + if not buff: + return + + # Possible bug? What if the 1024 cuts off in the middle of a utf8 + # code point? + # We use errors='replace' to have Python replace the unreadable + # character with an "official U+FFFD REPLACEMENT CHARACTER" + # This isn't great, but it's better than the previous behavior, + # which blew up on any issues. + buff = buff.decode(encoding='utf8', errors='replace') + + # An alternative is to try to read additional bytes one at a time + # until we can decode the string properly + # while True: + # try: + # buff = buff.decode(encoding='utf8') + # except UnicodeDecodeError: + # # Try to read another byte (this may not read anything) + # b = os.read(self.fd, 1) + # # If we read something + # if b: + # buff += b + # else: + # buff = buff.decode(encoding='utf8', errors='ignore') + # else: + # # If we could decode to UTF-8, then continue + # break + + # Append to previous buffer + if self.buffer: + buff = self.buffer + buff + self.buffer = '' + + lines = buff.splitlines(True) + # If the last character of the last line is not a newline + if lines[-1][-1] != '\n': # Incomplete line in the buffer + self.buffer = lines[-1] # Save the last line fragment + lines = lines[:-1] + + for line in lines: + self.handler(self.path, line[:-1]) + + def reopen(self, event=None, skip_to_end=False): + # stat the file on disk + file_stat = os.stat(self.path) + + # stat the file from the existing file descriptor + fd_stat = os.fstat(self.fd) + # Seek right back where we thought we were + pos = os.lseek(self.fd, 0, os.SEEK_CUR) + + # If the file now on disk is larger than where we were currently reading + if fd_stat.st_size > pos: + # More data to read - read as normal + self.read() + # If the file now on disk is smaller (eg: if the file is a freshly + # rotated log), or if its inode has changed + if self.stat.st_size > file_stat.st_size or \ + self.stat.st_ino != file_stat: + self.close() + # Since we already read the entirety of the previous file, we don't + # want to skip any of the new file's contents, so don't seek to the + # end, and try to read from it immediately + self.open(seek_to_end=False) + self.read() + + def open(self, seek_to_end=False): + self.stat = os.stat(self.path) + self.fd = os.open(self.path, os.O_RDONLY | os.O_NONBLOCK) + + if not self.read_all or seek_to_end: + os.lseek(self.fd, 0, os.SEEK_END) + + file_event_handler = FileEventHandler(callbacks={ + 'created': None, + 'modified': self.read, + 'moved': self.reopen, + 'deleted': self.reopen, + }) + self.watch = self.observer.schedule(file_event_handler, self.path) + + def close(self): + os.close(self.fd) + self.observer.unschedule(self.watch) + if self.buffer: + self.handler(self.path, self.buffer) + + +class TailManager(object): + def __init__(self, *args, **kwargs): + self.observer = Observer() + self.tails = {} + + def tail_file(self, path, handler, read_all=False): + if handler not in self.tails.setdefault(path, {}): + sft = SingleFileTail(path, handler, + read_all=read_all, observer=self.observer) + self.tails[path][handler] = sft + + def stop_tailing_file(self, path, handler): + tailed_file = self.tails.get(path, {}).pop(handler) + tailed_file.close() + # Amortize some cleanup while we're at it + if not self.tails.get(path): + self.tails.pop(path) + + def run(self): + self.start() + while True: + time.sleep(1) + + def start(self): + self.observer.start() + + def stop(self): + for handlers in self.tails.values(): + for tailed_file in handlers.values(): + tailed_file.close() + self.observer.stop() + self.observer.join() class FileWatchSensor(Sensor): - def __init__(self, sensor_service, config=None): - super(FileWatchSensor, self).__init__(sensor_service=sensor_service, - config=config) - self._trigger = None - self._logger = self._sensor_service.get_logger(__name__) - self._tail = None + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._stop = False + self.trigger = None + self.logger = self.sensor_service.get_logger(__name__) def setup(self): - self._tail = Tail(filenames=[]) - self._tail.handler = self._handle_line - self._tail.should_run = True + self.tail_manager = TailManager() def run(self): - self._tail.run() + self.tail_manager.run() + while not self._stop: + eventlet.sleep(60) def cleanup(self): - if self._tail: - self._tail.should_run = False - - try: - self._tail.notifier.stop() - except Exception: - pass + self._stop = True + self.tail_manager.stop() def add_trigger(self, trigger): file_path = trigger['parameters'].get('file_path', None) @@ -54,16 +218,13 @@ def add_trigger(self, trigger): self._logger.error('Received trigger type without "file_path" field.') return - self._trigger = trigger.get('ref', None) + self.trigger = trigger.get('ref', None) if not self._trigger: raise Exception('Trigger %s did not contain a ref.' % trigger) - # Wait a bit to avoid initialization race in logshipper library - eventlet.sleep(1.0) - - self._tail.add_file(filename=file_path) - self._logger.info('Added file "%s"' % (file_path)) + self.tail_manager.tail_file(file_path, self._handle_line) + self.logger.info('Added file "%s"' % (file_path)) def update_trigger(self, trigger): pass @@ -72,21 +233,31 @@ def remove_trigger(self, trigger): file_path = trigger['parameters'].get('file_path', None) if not file_path: - self._logger.error('Received trigger type without "file_path" field.') + self.logger.error('Received trigger type without "file_path" field.') return - self._tail.remove_file(filename=file_path) - self._trigger = None + self.tail_manager.stop_tailing_file(file_path, self._handle_line) + self.trigger = None - self._logger.info('Removed file "%s"' % (file_path)) + self.logger.info('Removed file "%s"' % (file_path)) def _handle_line(self, file_path, line): - trigger = self._trigger payload = { 'file_path': file_path, 'file_name': os.path.basename(file_path), 'line': line } self._logger.debug('Sending payload %s for trigger %s to sensor_service.', - payload, trigger) + payload, self.trigger) self.sensor_service.dispatch(trigger=trigger, payload=payload) + + +if __name__ == '__main__': + tm = TailManager() + tm.tail_file('test.py', handler=print) + tm.run() + + def halt(sig, frame): + tm.stop() + sys.exit(0) + signal.signal(signal.SIGINT, halt) From 0a2f6ae3c4c2c61460340212f134098f307d3065 Mon Sep 17 00:00:00 2001 From: blag Date: Sun, 29 Nov 2020 02:14:52 -0800 Subject: [PATCH 02/35] Remove logshipper and pyinotify from requirements files --- contrib/linux/requirements.txt | 4 +--- fixed-requirements.txt | 1 - requirements.txt | 3 +-- st2actions/in-requirements.txt | 3 +-- st2actions/requirements.txt | 3 +-- 5 files changed, 4 insertions(+), 10 deletions(-) diff --git a/contrib/linux/requirements.txt b/contrib/linux/requirements.txt index 597fda989a..e59495eee0 100644 --- a/contrib/linux/requirements.txt +++ b/contrib/linux/requirements.txt @@ -1,3 +1 @@ -# used by file watcher sensor -pyinotify>=0.9.5,<=0.10 --e git+https://github.com/StackStorm/logshipper.git@stackstorm_patched#egg=logshipper +watchdog diff --git a/fixed-requirements.txt b/fixed-requirements.txt index 0326841b1a..d82a248358 100644 --- a/fixed-requirements.txt +++ b/fixed-requirements.txt @@ -27,7 +27,6 @@ paramiko==2.7.1 passlib==1.7.1 prance==0.9.0 prompt-toolkit==1.0.15 -pyinotify==0.9.6; platform_system=="Linux" pymongo==3.10.0 python-editor==1.0.4 python-gnupg==0.4.5 diff --git a/requirements.txt b/requirements.txt index d2a7cee282..c24de018b9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,7 +14,6 @@ cryptography==3.2 dnspython<2.0.0,>=1.16.0 eventlet==0.25.1 flex==6.14.0 -git+https://github.com/StackStorm/logshipper.git@stackstorm_patched#egg=logshipper git+https://github.com/StackStorm/orquesta.git@v1.2.0#egg=orquesta git+https://github.com/StackStorm/st2-auth-backend-flat-file.git@master#egg=st2-auth-backend-flat-file git+https://github.com/StackStorm/st2-auth-ldap.git@master#egg=st2-auth-ldap @@ -42,7 +41,6 @@ passlib==1.7.1 prettytable prompt-toolkit==1.0.15 psutil==5.6.6 -pyinotify==0.9.6 ; platform_system == "Linux" pymongo==3.10.0 pyrabbit python-dateutil==2.8.0 @@ -65,6 +63,7 @@ stevedore==1.30.1 tooz==1.66.1 ujson==1.35 unittest2 +watchdog webob==1.8.5 webtest zake==0.2.2 diff --git a/st2actions/in-requirements.txt b/st2actions/in-requirements.txt index dcc00c6cf3..1ac704bfa8 100644 --- a/st2actions/in-requirements.txt +++ b/st2actions/in-requirements.txt @@ -14,7 +14,6 @@ python-json-logger gitpython lockfile # needed by core "linux" pack - TODO: create virtualenv for linux pack on postinst -pyinotify -git+https://github.com/StackStorm/logshipper.git@stackstorm_patched#egg=logshipper +watchdog # required by pack_mgmt/setup_virtualenv.py#L135 virtualenv diff --git a/st2actions/requirements.txt b/st2actions/requirements.txt index 80d3661f38..15d2712304 100755 --- a/st2actions/requirements.txt +++ b/st2actions/requirements.txt @@ -7,16 +7,15 @@ # update the component requirements.txt apscheduler==3.6.3 eventlet==0.25.1 -git+https://github.com/StackStorm/logshipper.git@stackstorm_patched#egg=logshipper gitpython==2.1.15 jinja2==2.10.3 kombu==4.6.6 lockfile==0.12.2 oslo.config<1.13,>=1.12.1 oslo.utils<=3.37.0,>=3.36.2 -pyinotify==0.9.6 ; platform_system == "Linux" python-dateutil==2.8.0 python-json-logger pyyaml==5.1.2 requests[security]==2.23.0 six==1.13.0 +watchdog From 2f71b2fcb8c5ec7b7b69c0e341ddd80c6b8c3638 Mon Sep 17 00:00:00 2001 From: blag Date: Sat, 12 Dec 2020 00:07:22 -0800 Subject: [PATCH 03/35] Import eventlet since we use it --- contrib/linux/sensors/file_watch_sensor.py | 1 + 1 file changed, 1 insertion(+) diff --git a/contrib/linux/sensors/file_watch_sensor.py b/contrib/linux/sensors/file_watch_sensor.py index a4e58ae0e5..4096487ed0 100644 --- a/contrib/linux/sensors/file_watch_sensor.py +++ b/contrib/linux/sensors/file_watch_sensor.py @@ -18,6 +18,7 @@ import time import sys +import eventlet from watchdog.observers import Observer from watchdog.events import FileSystemEventHandler From 3f0abb2d1e8255f07ae56cde0006589f1a8efc36 Mon Sep 17 00:00:00 2001 From: blag Date: Sat, 12 Dec 2020 00:07:47 -0800 Subject: [PATCH 04/35] Use self.logger instead of self._logger --- contrib/linux/sensors/file_watch_sensor.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/contrib/linux/sensors/file_watch_sensor.py b/contrib/linux/sensors/file_watch_sensor.py index 4096487ed0..418be52269 100644 --- a/contrib/linux/sensors/file_watch_sensor.py +++ b/contrib/linux/sensors/file_watch_sensor.py @@ -216,7 +216,7 @@ def add_trigger(self, trigger): file_path = trigger['parameters'].get('file_path', None) if not file_path: - self._logger.error('Received trigger type without "file_path" field.') + self.logger.error('Received trigger type without "file_path" field.') return self.trigger = trigger.get('ref', None) @@ -248,8 +248,8 @@ def _handle_line(self, file_path, line): 'file_name': os.path.basename(file_path), 'line': line } - self._logger.debug('Sending payload %s for trigger %s to sensor_service.', - payload, self.trigger) + self.logger.debug('Sending payload %s for trigger %s to sensor_service.', + payload, self.trigger) self.sensor_service.dispatch(trigger=trigger, payload=payload) From d175507ab047c81a95e6a01848b1928da93b633f Mon Sep 17 00:00:00 2001 From: blag Date: Sat, 12 Dec 2020 00:08:06 -0800 Subject: [PATCH 05/35] Use self.trigger instead of self._trigger --- contrib/linux/sensors/file_watch_sensor.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/contrib/linux/sensors/file_watch_sensor.py b/contrib/linux/sensors/file_watch_sensor.py index 418be52269..d428e0e794 100644 --- a/contrib/linux/sensors/file_watch_sensor.py +++ b/contrib/linux/sensors/file_watch_sensor.py @@ -221,7 +221,7 @@ def add_trigger(self, trigger): self.trigger = trigger.get('ref', None) - if not self._trigger: + if not self.trigger: raise Exception('Trigger %s did not contain a ref.' % trigger) self.tail_manager.tail_file(file_path, self._handle_line) @@ -250,7 +250,7 @@ def _handle_line(self, file_path, line): } self.logger.debug('Sending payload %s for trigger %s to sensor_service.', payload, self.trigger) - self.sensor_service.dispatch(trigger=trigger, payload=payload) + self.sensor_service.dispatch(trigger=self.trigger, payload=payload) if __name__ == '__main__': From 8add2ccb9e2e5331b94bdecdb269872c38428b96 Mon Sep 17 00:00:00 2001 From: blag Date: Tue, 22 Dec 2020 02:50:58 -0800 Subject: [PATCH 06/35] Refactor for testability and test coverage --- contrib/linux/sensors/file_watch_sensor.py | 256 +++++++++++++------ contrib/linux/sensors/file_watch_sensor.yaml | 5 + 2 files changed, 179 insertions(+), 82 deletions(-) diff --git a/contrib/linux/sensors/file_watch_sensor.py b/contrib/linux/sensors/file_watch_sensor.py index d428e0e794..17e2cd1e2a 100644 --- a/contrib/linux/sensors/file_watch_sensor.py +++ b/contrib/linux/sensors/file_watch_sensor.py @@ -13,7 +13,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +import functools import os +import pathlib import signal import time import sys @@ -28,12 +30,12 @@ Sensor = object -class FileEventHandler(FileSystemEventHandler): +class EventHandler(FileSystemEventHandler): def __init__(self, *args, callbacks=None, **kwargs): self.callbacks = callbacks or {} def dispatch(self, event): - if not event.is_synthetic and not event.is_directory: + if not event.is_directory: super().dispatch(event) def on_created(self, event): @@ -44,7 +46,7 @@ def on_created(self, event): def on_modified(self, event): cb = self.callbacks.get('modified') if cb: - cb(event=event) + result = cb(event=event) def on_moved(self, event): cb = self.callbacks.get('moved') @@ -58,47 +60,93 @@ def on_deleted(self, event): class SingleFileTail(object): - def __init__(self, path, handler, read_all=False, observer=None): - self.path = path + def __init__(self, path, handler, follow=False, read_all=False, observer=None, fd=None): + self._path = None + self.fd = fd self.handler = handler + self.follow = follow self.read_all = read_all self.buffer = '' self.observer = observer or Observer() - - self.open() + self.watch = None + self.parent_watch = None + + if path: + self.set_path(path) + self.open() + + def get_path(self): + return self._path + + # Set all of these when the path updates + def set_path(self, new_path): + self._path = pathlib.Path(new_path) + self.parent_dir = self.get_parent_path(self._path) + self.abs_path = (self.parent_dir / self._path).resolve() + + path = property(get_path, set_path) + + def get_parent_path(self, path): + if path.is_absolute(): + return pathlib.Path(path).parent + else: + return (pathlib.Path.cwd() / path).parent + + def get_event_src_path(self, event): + return (pathlib.Path.cwd() / pathlib.Path(event.src_path)).resolve() + + def read_chunk(self, fd, chunk_size=1024): + # Buffer 1024 bytes at a time + try: + buffer = os.read(fd, chunk_size) + except (OSError, FileNotFoundError): + buffer = b"" + + # If the 1024 bytes cuts the line off in the middle of a multi-byte + # utf-8 character then decoding will raise an UnicodeDecodeError. + try: + buffer = buffer.decode(encoding='utf8') + except UnicodeDecodeError as e: + # Grab the first few bytes of the partial character + # e.start is the first byte of the decoding issue + first_byte_of_partial_character = buffer[e.start] + number_of_bytes_read_so_far = e.end - e.start + + # Try to read the remainder of the character + # You could replace these conditionals with bit math, but that's a + # lot more difficult to read + if first_byte_of_partial_character & 0xF0 == 0xC0: + char_length = 2 + elif first_byte_of_partial_character & 0xF0 == 0xE0: + char_length = 3 + elif first_byte_of_partial_character & 0xF0 == 0xF0: + char_length = 4 + else: + # We could have run into an issue besides reading a partial + # character, so raise that exception + raise e + + number_of_bytes_to_read = char_length - number_of_bytes_read_so_far + + buff = os.read(fd, number_of_bytes_to_read) + if len(buff) == number_of_bytes_to_read: + buffer += buff + return buffer.decode(encoding='utf8') + + # If we did not successfully read a complete character, there's + # nothing else we can really do but reraise the exception + raise e + else: + return buffer def read(self, event=None): while True: - # Buffer 1024 bytes at a time - buff = os.read(self.fd, 1024) + # Read a chunk of bytes + buff = self.read_chunk(self.fd) + if not buff: return - # Possible bug? What if the 1024 cuts off in the middle of a utf8 - # code point? - # We use errors='replace' to have Python replace the unreadable - # character with an "official U+FFFD REPLACEMENT CHARACTER" - # This isn't great, but it's better than the previous behavior, - # which blew up on any issues. - buff = buff.decode(encoding='utf8', errors='replace') - - # An alternative is to try to read additional bytes one at a time - # until we can decode the string properly - # while True: - # try: - # buff = buff.decode(encoding='utf8') - # except UnicodeDecodeError: - # # Try to read another byte (this may not read anything) - # b = os.read(self.fd, 1) - # # If we read something - # if b: - # buff += b - # else: - # buff = buff.decode(encoding='utf8', errors='ignore') - # else: - # # If we could decode to UTF-8, then continue - # break - # Append to previous buffer if self.buffer: buff = self.buffer + buff @@ -106,68 +154,109 @@ def read(self, event=None): lines = buff.splitlines(True) # If the last character of the last line is not a newline - if lines[-1][-1] != '\n': # Incomplete line in the buffer + if lines and lines[-1] and lines[-1][-1] != '\n': # Incomplete line in the buffer self.buffer = lines[-1] # Save the last line fragment lines = lines[:-1] for line in lines: self.handler(self.path, line[:-1]) - def reopen(self, event=None, skip_to_end=False): - # stat the file on disk - file_stat = os.stat(self.path) + def reopen_and_read(self, event=None, skip_to_end=False): + # Directory watches will fire events for unrelated files + # Ignore all events except those for our path + if event and self.get_event_src_path(event) != self.abs_path: + return - # stat the file from the existing file descriptor - fd_stat = os.fstat(self.fd) - # Seek right back where we thought we were + # Save our current position into the file (this is a little wonky) pos = os.lseek(self.fd, 0, os.SEEK_CUR) - # If the file now on disk is larger than where we were currently reading - if fd_stat.st_size > pos: - # More data to read - read as normal - self.read() - # If the file now on disk is smaller (eg: if the file is a freshly - # rotated log), or if its inode has changed - if self.stat.st_size > file_stat.st_size or \ - self.stat.st_ino != file_stat: - self.close() - # Since we already read the entirety of the previous file, we don't - # want to skip any of the new file's contents, so don't seek to the - # end, and try to read from it immediately - self.open(seek_to_end=False) - self.read() - - def open(self, seek_to_end=False): - self.stat = os.stat(self.path) - self.fd = os.open(self.path, os.O_RDONLY | os.O_NONBLOCK) - - if not self.read_all or seek_to_end: - os.lseek(self.fd, 0, os.SEEK_END) - - file_event_handler = FileEventHandler(callbacks={ - 'created': None, - 'modified': self.read, - 'moved': self.reopen, - 'deleted': self.reopen, - }) - self.watch = self.observer.schedule(file_event_handler, self.path) - - def close(self): - os.close(self.fd) - self.observer.unschedule(self.watch) - if self.buffer: + # The file was moved and not recreated + # If we're following the file, don't emit the remainder of the last + # line + emit = not self.follow + self.close(event=event, emit_remaining=emit) + + # If we aren't following then don't reopen the file + # When the file is created again that will be handled by open_and_read + if not self.follow: + return + + # Use the file's new location + self.path = event.dest_path + # Seek to where we left off + self.open(event=event, seek_to=pos) + self.read(event=event) + + def open_and_read(self, event=None, seek_to=None): + # Directory watches will fire events for unrelated files + # Ignore all events except those for our path + if event and self.get_event_src_path(event) != self.abs_path: + return + + self.read_all = True + + self.open(event=event, seek_to=seek_to) + self.read(event=event) + + def open(self, event=None, seek_to=None): + # Use self.watch as a guard + if not self.watch: + try: + self.stat = os.stat(self.path) + except FileNotFoundError: + # If the file doesn't exist when we are asked to monitor it, set + # this flag so we read it all if/when it does appear + self.read_all = True + else: + self.fd = os.open(self.path, os.O_RDONLY | os.O_NONBLOCK) + + if self.read_all or seek_to == 'start': + os.lseek(self.fd, 0, os.SEEK_SET) + + if not self.read_all or seek_to == 'end': + os.lseek(self.fd, 0, os.SEEK_END) + + file_event_handler = EventHandler(callbacks={ + 'created': self.open, + 'deleted': self.close, + 'modified': self.read, + 'moved': self.reopen_and_read, + }) + + self.watch = self.observer.schedule(file_event_handler, self.path) + + # Avoid watching this twice + if not self.parent_watch: + dir_event_handler = EventHandler(callbacks={ + 'created': self.open_and_read, + 'moved': self.reopen_and_read, + }) + + self.parent_watch = self.observer.schedule(dir_event_handler, self.parent_dir) + + def close(self, event=None, emit_remaining=True): + # Reset the guard + if self.buffer and emit_remaining: self.handler(self.path, self.buffer) + self.buffer = '' + if self.fd: + os.close(self.fd) + self.fd = None + if self.watch: + self.observer.unschedule(self.watch) + self.watch = None class TailManager(object): def __init__(self, *args, **kwargs): - self.observer = Observer() self.tails = {} + self.observer = Observer() - def tail_file(self, path, handler, read_all=False): + def tail_file(self, path, handler, follow=False, read_all=False): if handler not in self.tails.setdefault(path, {}): sft = SingleFileTail(path, handler, - read_all=read_all, observer=self.observer) + follow=follow, read_all=read_all, + observer=self.observer) self.tails[path][handler] = sft def stop_tailing_file(self, path, handler): @@ -179,8 +268,11 @@ def stop_tailing_file(self, path, handler): def run(self): self.start() - while True: - time.sleep(1) + try: + while True: + time.sleep(1) + finally: + self.stop() def start(self): self.observer.start() @@ -245,7 +337,7 @@ def remove_trigger(self, trigger): def _handle_line(self, file_path, line): payload = { 'file_path': file_path, - 'file_name': os.path.basename(file_path), + 'file_name': pathlib.Path(file_path).name, 'line': line } self.logger.debug('Sending payload %s for trigger %s to sensor_service.', @@ -255,7 +347,7 @@ def _handle_line(self, file_path, line): if __name__ == '__main__': tm = TailManager() - tm.tail_file('test.py', handler=print) + tm.tail_file(__file__, handler=print) tm.run() def halt(sig, frame): diff --git a/contrib/linux/sensors/file_watch_sensor.yaml b/contrib/linux/sensors/file_watch_sensor.yaml index ba622a9eb7..0550d3e29d 100644 --- a/contrib/linux/sensors/file_watch_sensor.yaml +++ b/contrib/linux/sensors/file_watch_sensor.yaml @@ -16,6 +16,11 @@ description: "Path to the file to monitor" type: "string" required: true + follow: + description: "Whether or not to follow the file when it moves" + type: "boolean" + required: false + default: false additionalProperties: false # This is the schema of the trigger payload the sensor generates payload_schema: From 169cc5ac04abdf25780c6b62c38a179f75cc5034 Mon Sep 17 00:00:00 2001 From: blag Date: Tue, 22 Dec 2020 11:15:26 -0800 Subject: [PATCH 07/35] Add tests for FileWatchSensor components --- contrib/linux/tests/test_file_watch_sensor.py | 688 ++++++++++++++++++ 1 file changed, 688 insertions(+) create mode 100644 contrib/linux/tests/test_file_watch_sensor.py diff --git a/contrib/linux/tests/test_file_watch_sensor.py b/contrib/linux/tests/test_file_watch_sensor.py new file mode 100644 index 0000000000..a610c6b600 --- /dev/null +++ b/contrib/linux/tests/test_file_watch_sensor.py @@ -0,0 +1,688 @@ +import functools +import os +import pathlib +import shutil +import tempfile +import time +import unittest + +import mock + +from file_watch_sensor import SingleFileTail, TailManager + +WAIT_TIME = 0.1 + + +def test_single_file_tail_read_chunk_over_multibyte_character_boundary(): + wide_characters = [None, None, '\u0130', '\u2050', '\U00088080'] + for n in range(2, 5): + yield from _gen_n_byte_character_tests(1024, n, wide_characters[n]) + + for n in range(2, 5): + yield from _gen_n_byte_character_tests(2048, n, wide_characters[n]) + + for n in range(2, 5): + yield from _gen_n_byte_character_tests(4096, n, wide_characters[n]) + + +def _gen_n_byte_character_tests(chunk_size, n, char): + for length in range(chunk_size, chunk_size + n + 1): + yield _run_n_byte_character_tests, chunk_size, n, length, char + + +def _run_n_byte_character_tests(chunk_size, n, length, char): + filename = f'chunk_boundary_{n}u_{length}.txt' + + with open(filename, 'wb+') as f: + # Write out a file that is of the given length + # aaaaaa...aaa\x82 + f.write(('a' * (length - n) + char).encode('utf-8')) + + fd = os.open(filename, os.O_RDONLY) + + sft = SingleFileTail(None, None, fd=fd) + + result = sft.read_chunk(fd, chunk_size=chunk_size) + + os.close(fd) + os.unlink(filename) + + if length < chunk_size + n: + assert result == ('a' * (length - n) + char) + else: + assert result == ('a' * chunk_size) + + +def test_single_file_tail_read_chunk_with_bad_utf8_character(): + filename = f'bad_utf8_character.txt' + + utf8_str = '\U00088080' + utf8_bytes = utf8_str.encode('utf-8') + chopped_utf8_bytes = utf8_bytes[:-1] + + with open(filename, 'wb+') as f: + # Write out a file that is of the given length + # aaaaaa...aaa\x82 + f.write(b'a') + f.write(chopped_utf8_bytes) + f.write(b'a') + + fd = os.open(filename, os.O_RDONLY) + + sft = SingleFileTail(None, None, fd=fd) + + err = None + try: + result = sft.read_chunk(fd) + except Exception as e: + err = e + finally: + assert err is not None + assert isinstance(err, UnicodeDecodeError) + + os.close(fd) + os.unlink(filename) + + +def test_single_file_tail_append_to_watched_file_with_absolute_path(): + tailed_filename = (pathlib.Path.cwd() / pathlib.Path('tailed_file.txt')).resolve() + with open(tailed_filename, 'w+') as f: + f.write("Preexisting text line 1\n") + f.write("Preexisting text line 2\n") + + appended_lines = [] + + def append_to_list(list_to_append, path, element): + list_to_append.append(element) + + append_to_list_partial = functools.partial(append_to_list, appended_lines) + + tm = TailManager() + tm.tail_file(tailed_filename, handler=append_to_list_partial) + tm.start() + + with open(tailed_filename, 'a+') as f: + f.write("Added line 1\n") + time.sleep(WAIT_TIME) + + assert appended_lines == [ + "Added line 1", + ] + + tm.stop_tailing_file(tailed_filename, handler=append_to_list_partial) + + tm.stop() + + os.unlink(tailed_filename) + + +def test_single_file_tail_not_watched_file(): + tailed_filename = 'tailed_file.txt' + not_tailed_filename = 'not_tailed_file.txt' + new_not_tailed_filename = not_tailed_filename.replace('.txt', '_moved.txt') + with open(tailed_filename, 'w+') as f: + f.write("Preexisting text line 1\n") + f.write("Preexisting text line 2\n") + + appended_lines = [] + + def append_to_list(list_to_append, path, element): + list_to_append.append(element) + + append_to_list_partial = functools.partial(append_to_list, appended_lines) + + tm = TailManager() + tm.tail_file(tailed_filename, handler=append_to_list_partial) + tm.start() + + with open(tailed_filename, 'a+') as f: + f.write("Added line 1\n") + time.sleep(WAIT_TIME) + + assert appended_lines == [ + "Added line 1", + ] + + with open(not_tailed_filename, 'a+') as f: + f.write("Added line 1 - not tailed\n") + time.sleep(WAIT_TIME) + + assert appended_lines == [ + "Added line 1", + ] + + os.replace(not_tailed_filename, new_not_tailed_filename) + time.sleep(WAIT_TIME) + + assert appended_lines == [ + "Added line 1", + ] + + tm.stop_tailing_file(tailed_filename, handler=append_to_list_partial) + + tm.stop() + time.sleep(WAIT_TIME) + + assert appended_lines == [ + "Added line 1", + ] + + os.unlink(tailed_filename) + os.unlink(new_not_tailed_filename) + + +def test_single_file_tail_watch_nonexistent_file(): + tailed_filename = 'tailed_file.txt' + other_tailed_filename = f'other_{tailed_filename}' + + if os.path.exists(tailed_filename): + os.unlink(tailed_filename) + + appended_lines = [] + + def append_to_list(list_to_append, path, element): + list_to_append.append(element) + + append_to_list_partial = functools.partial(append_to_list, appended_lines) + + tm = TailManager() + tm.tail_file(tailed_filename, handler=append_to_list_partial) + tm.start() + time.sleep(WAIT_TIME) + + assert appended_lines == [] + + with open(tailed_filename, 'w+') as f: + f.write("Added line 1\n") + time.sleep(WAIT_TIME) + + assert appended_lines == [ + "Added line 1", + ] + + tm.stop_tailing_file(tailed_filename, handler=append_to_list_partial) + + tm.stop() + time.sleep(WAIT_TIME) + + os.unlink(tailed_filename) + + +def test_single_file_tail_follow_watched_file_moved(): + tailed_filename = 'tailed_file_to_move.txt' + new_filename = tailed_filename.replace('_to_move.txt', '_moved.txt') + + if os.path.exists(new_filename): + os.unlink(new_filename) + if os.path.exists(tailed_filename): + os.unlink(tailed_filename) + + with open(tailed_filename, 'w+') as f: + f.write("Preexisting text line 1\n") + f.write("Preexisting text line 2\n") + + appended_lines = [] + + def append_to_list(list_to_append, path, element): + list_to_append.append(element) + + append_to_list_partial = functools.partial(append_to_list, appended_lines) + + tm = TailManager() + tm.tail_file(tailed_filename, handler=append_to_list_partial, follow=True) + tm.start() + + with open(tailed_filename, 'a+') as f: + f.write("Added line 1\n") + time.sleep(WAIT_TIME) + + assert appended_lines == [ + "Added line 1", + ] + + with open(tailed_filename, 'a+') as f: + f.write("Added line 2") # No newline + time.sleep(WAIT_TIME) + + assert appended_lines == [ + "Added line 1", + ] + + os.replace(tailed_filename, new_filename) + time.sleep(WAIT_TIME) + + assert appended_lines == [ + "Added line 1", + ] + + with open(new_filename, 'a+') as f: + f.write(" - end of line 2\n") + time.sleep(WAIT_TIME) + + assert appended_lines == [ + "Added line 1", + "Added line 2 - end of line 2", + ] + + with open(tailed_filename, 'w+') as f: + f.write("New file - text line 1\n") + f.write("New file - text line 2\n") + time.sleep(WAIT_TIME) + + assert appended_lines == [ + "Added line 1", + "Added line 2 - end of line 2", + ] + + tm.stop_tailing_file(tailed_filename, handler=append_to_list_partial) + + tm.stop() + + os.unlink(new_filename) + os.unlink(tailed_filename) + + +def test_single_file_tail_not_followed_watched_file_moved(): + tailed_filename = 'tailed_file_to_move.txt' + new_filename = tailed_filename.replace('_to_move.txt', '_moved.txt') + + if os.path.exists(new_filename): + os.unlink(new_filename) + if os.path.exists(tailed_filename): + os.unlink(tailed_filename) + + with open(tailed_filename, 'w+') as f: + f.write("Preexisting text line 1\n") + f.write("Preexisting text line 2\n") + + appended_lines = [] + + def append_to_list(list_to_append, path, element): + list_to_append.append(element) + + append_to_list_partial = functools.partial(append_to_list, appended_lines) + + tm = TailManager() + tm.tail_file(tailed_filename, handler=append_to_list_partial, follow=False) + tm.start() + + with open(tailed_filename, 'a+') as f: + f.write("Added line 1\n") + time.sleep(WAIT_TIME) + + assert appended_lines == [ + "Added line 1", + ] + + with open(tailed_filename, 'a+') as f: + f.write("Added line 2") # No newline + time.sleep(WAIT_TIME) + + assert appended_lines == [ + "Added line 1", + ] + + os.replace(tailed_filename, new_filename) + time.sleep(WAIT_TIME) + + assert appended_lines == [ + "Added line 1", + "Added line 2", + ] + + with open(new_filename, 'a+') as f: + f.write(" - end of line 2\n") + time.sleep(WAIT_TIME) + + assert appended_lines == [ + "Added line 1", + "Added line 2", + ] + + with open(tailed_filename, 'w+') as f: + f.write("Recreated file - text line 1\n") + f.write("Recreated file - text line 2\n") + time.sleep(WAIT_TIME) + + assert appended_lines == [ + "Added line 1", + "Added line 2", + "Recreated file - text line 1", + "Recreated file - text line 2", + ] + + tm.stop_tailing_file(tailed_filename, handler=append_to_list_partial) + + tm.stop() + + os.unlink(new_filename) + os.unlink(tailed_filename) + + +def test_single_file_tail_non_watched_file_moved(): + tailed_filename = 'tailed_file_to_move.txt' + not_tailed_filename = f'not_{tailed_filename}' + new_not_tailed_filename = not_tailed_filename.replace('_to_move.txt', '_moved.txt') + + if os.path.exists(not_tailed_filename): + os.unlink(not_tailed_filename) + if os.path.exists(new_not_tailed_filename): + os.unlink(new_not_tailed_filename) + if os.path.exists(tailed_filename): + os.unlink(tailed_filename) + + with open(not_tailed_filename, 'w+') as f: + f.write("Text here will not be monitored\n") + + with open(tailed_filename, 'w+') as f: + f.write("Preexisting text line 1\n") + f.write("Preexisting text line 2\n") + + appended_lines = [] + + def append_to_list(list_to_append, path, element): + list_to_append.append(element) + + append_to_list_partial = functools.partial(append_to_list, appended_lines) + + tm = TailManager() + tm.tail_file(tailed_filename, handler=append_to_list_partial) + tm.start() + + with open(tailed_filename, 'a+') as f: + f.write("Added line 1\n") + time.sleep(WAIT_TIME) + + assert appended_lines == [ + "Added line 1", + ] + + os.replace(not_tailed_filename, new_not_tailed_filename) + time.sleep(WAIT_TIME) + + assert appended_lines == [ + "Added line 1", + ] + + tm.stop_tailing_file(tailed_filename, handler=append_to_list_partial) + + tm.stop() + + os.unlink(new_not_tailed_filename) + os.unlink(tailed_filename) + + +def test_single_file_tail_watched_file_deleted(): + tailed_filename = 'tailed_file_deleted.txt' + with open(tailed_filename, 'w+') as f: + f.write("Preexisting text line 1\n") + f.write("Preexisting text line 2\n") + + appended_lines = [] + + def append_to_list(list_to_append, path, element): + list_to_append.append(element) + + append_to_list_partial = functools.partial(append_to_list, appended_lines) + + tm = TailManager() + tm.tail_file(tailed_filename, handler=append_to_list_partial) + tm.start() + + os.unlink(tailed_filename) + + tm.stop() + + +def test_tail_manager_append_to_watched_file(): + tailed_filename = 'tailed_file.txt' + with open(tailed_filename, 'w+') as f: + f.write("Preexisting text line 1\n") + f.write("Preexisting text line 2\n") + + appended_lines = [] + + def append_to_list(list_to_append, path, element): + list_to_append.append(element) + + append_to_list_partial = functools.partial(append_to_list, appended_lines) + + tm = TailManager() + tm.tail_file(tailed_filename, handler=append_to_list_partial) + tm.start() + + with open(tailed_filename, 'a+') as f: + f.write("Added line 1\n") + time.sleep(WAIT_TIME) + + assert appended_lines == [ + "Added line 1", + ] + + with open(tailed_filename, 'a+') as f: + f.write("Added line 2\n") + time.sleep(WAIT_TIME) + + assert appended_lines == [ + "Added line 1", + "Added line 2", + ] + + with open(tailed_filename, 'a+') as f: + f.write("Start of added partial line 1") + time.sleep(WAIT_TIME) + + assert appended_lines == [ + "Added line 1", + "Added line 2", + ] + + with open(tailed_filename, 'a+') as f: + f.write(" - finished partial line 1\nStart of added partial line 2") + time.sleep(WAIT_TIME) + + assert appended_lines == [ + "Added line 1", + "Added line 2", + "Start of added partial line 1 - finished partial line 1", + ] + + with open(tailed_filename, 'a+') as f: + f.write(" - finished partial line 2\n") + time.sleep(WAIT_TIME) + + assert appended_lines == [ + "Added line 1", + "Added line 2", + "Start of added partial line 1 - finished partial line 1", + "Start of added partial line 2 - finished partial line 2", + ] + + with open(tailed_filename, 'a+') as f: + f.write("Final line without a newline") + time.sleep(WAIT_TIME) + + tm.stop_tailing_file(tailed_filename, handler=append_to_list_partial) + + tm.stop() + + time.sleep(WAIT_TIME) + assert appended_lines == [ + "Added line 1", + "Added line 2", + "Start of added partial line 1 - finished partial line 1", + "Start of added partial line 2 - finished partial line 2", + "Final line without a newline", + ] + + os.unlink(tailed_filename) + + +def test_tail_manager_tail_file_twice(): + tailed_filename = 'tailed_file.txt' + with open(tailed_filename, 'w+') as f: + f.write("Preexisting text line 1\n") + f.write("Preexisting text line 2\n") + + appended_lines = [] + + def append_to_list(list_to_append, path, element): + list_to_append.append(element) + + append_to_list_partial = functools.partial(append_to_list, appended_lines) + + tm = TailManager() + tm.tail_file(tailed_filename, handler=append_to_list_partial) + tm.tail_file(tailed_filename, handler=append_to_list_partial) + tm.start() + + with open(tailed_filename, 'a+') as f: + f.write("Added line 1\n") + time.sleep(WAIT_TIME) + + assert appended_lines == [ + "Added line 1", + ] + + with open(tailed_filename, 'a+') as f: + f.write("Added line 2\n") + time.sleep(WAIT_TIME) + + assert appended_lines == [ + "Added line 1", + "Added line 2", + ] + + with open(tailed_filename, 'a+') as f: + f.write("Start of added partial line 1") + time.sleep(WAIT_TIME) + + assert appended_lines == [ + "Added line 1", + "Added line 2", + ] + + with open(tailed_filename, 'a+') as f: + f.write(" - finished partial line 1\nStart of added partial line 2") + time.sleep(WAIT_TIME) + + assert appended_lines == [ + "Added line 1", + "Added line 2", + "Start of added partial line 1 - finished partial line 1", + ] + + with open(tailed_filename, 'a+') as f: + f.write(" - finished partial line 2\n") + time.sleep(WAIT_TIME) + + assert appended_lines == [ + "Added line 1", + "Added line 2", + "Start of added partial line 1 - finished partial line 1", + "Start of added partial line 2 - finished partial line 2", + ] + + with open(tailed_filename, 'a+') as f: + f.write("Final line without a newline") + time.sleep(WAIT_TIME) + + tm.stop_tailing_file(tailed_filename, handler=append_to_list_partial) + + tm.stop() + + time.sleep(WAIT_TIME) + assert appended_lines == [ + "Added line 1", + "Added line 2", + "Start of added partial line 1 - finished partial line 1", + "Start of added partial line 2 - finished partial line 2", + "Final line without a newline", + ] + + os.unlink(tailed_filename) + + +def test_tail_manager_stop(): + tailed_filename = 'tailed_file_stop.txt' + with open(tailed_filename, 'w+') as f: + f.write("Preexisting text line 1\n") + f.write("Preexisting text line 2\n") + + appended_lines = [] + + def append_to_list(list_to_append, path, element): + list_to_append.append(element) + + append_to_list_partial = functools.partial(append_to_list, appended_lines) + + tm = TailManager() + tm.tail_file(tailed_filename, handler=append_to_list_partial) + tm.start() + + with open(tailed_filename, 'a+') as f: + f.write("Added line 1\n") + time.sleep(WAIT_TIME) + + assert appended_lines == [ + "Added line 1", + ] + + with open(tailed_filename, 'a+') as f: + f.write("Final line without a newline") + time.sleep(WAIT_TIME) + + tm.stop() + + time.sleep(WAIT_TIME) + assert appended_lines == [ + "Added line 1", + "Final line without a newline", + ] + + os.unlink(tailed_filename) + + +def test_tail_manager_stop_twice(): + tailed_filename = 'tailed_file_stop.txt' + with open(tailed_filename, 'w+') as f: + f.write("Preexisting text line 1\n") + f.write("Preexisting text line 2\n") + + appended_lines = [] + + def append_to_list(list_to_append, path, element): + list_to_append.append(element) + + append_to_list_partial = functools.partial(append_to_list, appended_lines) + + tm = TailManager() + tm.tail_file(tailed_filename, handler=append_to_list_partial) + tm.start() + + with open(tailed_filename, 'a+') as f: + f.write("Added line 1\n") + time.sleep(WAIT_TIME) + + assert appended_lines == [ + "Added line 1", + ] + + with open(tailed_filename, 'a+') as f: + f.write("Final line without a newline") + time.sleep(WAIT_TIME) + + tm.stop() + tm.stop() + + time.sleep(WAIT_TIME) + assert appended_lines == [ + "Added line 1", + "Final line without a newline", + ] + + os.unlink(tailed_filename) + + +if __name__ == '__main__': + test_single_file_tail_not_followed_watched_file_moved() From fa68bef52bfec1acba536b617a3b04274877dc0b Mon Sep 17 00:00:00 2001 From: blag Date: Wed, 23 Dec 2020 00:47:05 -0800 Subject: [PATCH 08/35] Linting --- contrib/linux/sensors/file_watch_sensor.py | 3 +-- contrib/linux/tests/test_file_watch_sensor.py | 26 +++++++++++++------ 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/contrib/linux/sensors/file_watch_sensor.py b/contrib/linux/sensors/file_watch_sensor.py index 17e2cd1e2a..d9b3af63a7 100644 --- a/contrib/linux/sensors/file_watch_sensor.py +++ b/contrib/linux/sensors/file_watch_sensor.py @@ -13,7 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import functools import os import pathlib import signal @@ -46,7 +45,7 @@ def on_created(self, event): def on_modified(self, event): cb = self.callbacks.get('modified') if cb: - result = cb(event=event) + cb(event=event) def on_moved(self, event): cb = self.callbacks.get('moved') diff --git a/contrib/linux/tests/test_file_watch_sensor.py b/contrib/linux/tests/test_file_watch_sensor.py index a610c6b600..e6767459be 100644 --- a/contrib/linux/tests/test_file_watch_sensor.py +++ b/contrib/linux/tests/test_file_watch_sensor.py @@ -1,12 +1,23 @@ +#!/usr/bin/env python + +# Copyright 2020 The StackStorm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import functools import os import pathlib -import shutil -import tempfile import time -import unittest - -import mock from file_watch_sensor import SingleFileTail, TailManager @@ -54,7 +65,7 @@ def _run_n_byte_character_tests(chunk_size, n, length, char): def test_single_file_tail_read_chunk_with_bad_utf8_character(): - filename = f'bad_utf8_character.txt' + filename = 'bad_utf8_character.txt' utf8_str = '\U00088080' utf8_bytes = utf8_str.encode('utf-8') @@ -73,7 +84,7 @@ def test_single_file_tail_read_chunk_with_bad_utf8_character(): err = None try: - result = sft.read_chunk(fd) + sft.read_chunk(fd) except Exception as e: err = e finally: @@ -173,7 +184,6 @@ def append_to_list(list_to_append, path, element): def test_single_file_tail_watch_nonexistent_file(): tailed_filename = 'tailed_file.txt' - other_tailed_filename = f'other_{tailed_filename}' if os.path.exists(tailed_filename): os.unlink(tailed_filename) From e932ffef3b94fa4112c609ae1a50889c38b6ba23 Mon Sep 17 00:00:00 2001 From: blag Date: Sun, 17 Jan 2021 01:04:01 -0700 Subject: [PATCH 09/35] Add class docstrings --- contrib/linux/sensors/file_watch_sensor.py | 36 ++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/contrib/linux/sensors/file_watch_sensor.py b/contrib/linux/sensors/file_watch_sensor.py index d9b3af63a7..f7fb2c47c8 100644 --- a/contrib/linux/sensors/file_watch_sensor.py +++ b/contrib/linux/sensors/file_watch_sensor.py @@ -30,6 +30,12 @@ class EventHandler(FileSystemEventHandler): + """ + A class to track and route different events to event handlers/callbacks for + files. This allows this EventHandler class to be used for watches on + individual files, since the directory events will include events for + individual files. + """ def __init__(self, *args, callbacks=None, **kwargs): self.callbacks = callbacks or {} @@ -59,6 +65,36 @@ def on_deleted(self, event): class SingleFileTail(object): + """ + A class to tail a single file, also handling emitting events when the + watched file is created, truncated, or moved. + + If follow is False (the default), then the watch will be removed when the + file is moved, and recreated if/when the file is recreated (with read_all + set to True so each line in the recreated file is handled). This mode + should be useful for logs that are rotated regularly. + + If follow is True, then the cursor position for the old file location will + be saved, the watch for the old file location will be removed, a new watch + for the new file location will be created, and only new lines added after + the previous cursor position will be handled. This mode should be useful + for user files that may be moved or renamed as they are being edited. + + If read_all is False (the default), then the file cursor will be set to the + end of the file and only new lines added after the watch is created will be + handled. This should be useful when you are only interested in lines that + are added to an already existing file while it is watched and you are not + interested in the contents of the file before it is watched. + + If read_all is True, then each line in the file, starting from the + beginning of the file, is handled. This should be useful when you wish to + fully process a file once it is created. + + Note that while the watch events are serialized in a queue, this code does + not attempt to serialize its own file access with locks, so a situation + where one file is quickly created and/or updated may trigger race + conditions and therefore unpredictable behavior. + """ def __init__(self, path, handler, follow=False, read_all=False, observer=None, fd=None): self._path = None self.fd = fd From 5a22c27f3bce94b9fa896fbf450b11a7faf3f3de Mon Sep 17 00:00:00 2001 From: blag Date: Sun, 17 Jan 2021 01:17:26 -0700 Subject: [PATCH 10/35] Add logging --- contrib/linux/sensors/file_watch_sensor.py | 58 ++++++++++++++++--- contrib/linux/tests/test_file_watch_sensor.py | 53 ++++++++++++----- 2 files changed, 91 insertions(+), 20 deletions(-) diff --git a/contrib/linux/sensors/file_watch_sensor.py b/contrib/linux/sensors/file_watch_sensor.py index f7fb2c47c8..8151010634 100644 --- a/contrib/linux/sensors/file_watch_sensor.py +++ b/contrib/linux/sensors/file_watch_sensor.py @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import logging import os import pathlib import signal @@ -95,7 +96,11 @@ class SingleFileTail(object): where one file is quickly created and/or updated may trigger race conditions and therefore unpredictable behavior. """ - def __init__(self, path, handler, follow=False, read_all=False, observer=None, fd=None): + def __init__(self, path, handler, follow=False, read_all=False, + observer=None, logger=None, fd=None): + if logger is None: + raise Exception("SingleFileTail was initialized without a logger") + self._path = None self.fd = fd self.handler = handler @@ -103,6 +108,7 @@ def __init__(self, path, handler, follow=False, read_all=False, observer=None, f self.read_all = read_all self.buffer = '' self.observer = observer or Observer() + self.logger = logger self.watch = None self.parent_watch = None @@ -115,6 +121,7 @@ def get_path(self): # Set all of these when the path updates def set_path(self, new_path): + self.logger.debug(f"Setting path to {new_path}") self._path = pathlib.Path(new_path) self.parent_dir = self.get_parent_path(self._path) self.abs_path = (self.parent_dir / self._path).resolve() @@ -131,11 +138,14 @@ def get_event_src_path(self, event): return (pathlib.Path.cwd() / pathlib.Path(event.src_path)).resolve() def read_chunk(self, fd, chunk_size=1024): + self.logger.debug("Reading chunk") # Buffer 1024 bytes at a time try: buffer = os.read(fd, chunk_size) except (OSError, FileNotFoundError): buffer = b"" + else: + self.logger.debug("Read chunk") # If the 1024 bytes cuts the line off in the middle of a multi-byte # utf-8 character then decoding will raise an UnicodeDecodeError. @@ -146,6 +156,7 @@ def read_chunk(self, fd, chunk_size=1024): # e.start is the first byte of the decoding issue first_byte_of_partial_character = buffer[e.start] number_of_bytes_read_so_far = e.end - e.start + self.logger.debug(f"Read {number_of_bytes_read_so_far}") # Try to read the remainder of the character # You could replace these conditionals with bit math, but that's a @@ -163,6 +174,7 @@ def read_chunk(self, fd, chunk_size=1024): number_of_bytes_to_read = char_length - number_of_bytes_read_so_far + self.logger.debug(f"Reading {number_of_bytes_to_read} more bytes") buff = os.read(fd, number_of_bytes_to_read) if len(buff) == number_of_bytes_to_read: buffer += buff @@ -175,6 +187,7 @@ def read_chunk(self, fd, chunk_size=1024): return buffer def read(self, event=None): + self.logger.debug("Reading file") while True: # Read a chunk of bytes buff = self.read_chunk(self.fd) @@ -184,26 +197,31 @@ def read(self, event=None): # Append to previous buffer if self.buffer: + self.logger.debug(f"Appending to existing buffer: '{self.buffer}'") buff = self.buffer + buff self.buffer = '' lines = buff.splitlines(True) # If the last character of the last line is not a newline if lines and lines[-1] and lines[-1][-1] != '\n': # Incomplete line in the buffer + self.logger.debug(f"Saving partial line in the buffer: '{lines[-1]}'") self.buffer = lines[-1] # Save the last line fragment lines = lines[:-1] for line in lines: + self.logger.debug(f"Passing line to callback: '{line[:-1]}'") self.handler(self.path, line[:-1]) def reopen_and_read(self, event=None, skip_to_end=False): # Directory watches will fire events for unrelated files # Ignore all events except those for our path if event and self.get_event_src_path(event) != self.abs_path: + self.logger.debug(f"Ignoring event for non-tracked file: '{event.src_path}'") return # Save our current position into the file (this is a little wonky) pos = os.lseek(self.fd, 0, os.SEEK_CUR) + self.logger.debug(f"Saving position ({pos}) into file {self.abs_path}") # The file was moved and not recreated # If we're following the file, don't emit the remainder of the last @@ -226,6 +244,7 @@ def open_and_read(self, event=None, seek_to=None): # Directory watches will fire events for unrelated files # Ignore all events except those for our path if event and self.get_event_src_path(event) != self.abs_path: + self.logger.debug(f"Ignoring event for non-tailed file: '{event.src_path}'") return self.read_all = True @@ -236,19 +255,23 @@ def open_and_read(self, event=None, seek_to=None): def open(self, event=None, seek_to=None): # Use self.watch as a guard if not self.watch: + self.logger.debug(f"Opening file '{self.path}'") try: self.stat = os.stat(self.path) except FileNotFoundError: # If the file doesn't exist when we are asked to monitor it, set # this flag so we read it all if/when it does appear + self.logger.debug("File does not yet exist, setting read_all=True") self.read_all = True else: self.fd = os.open(self.path, os.O_RDONLY | os.O_NONBLOCK) if self.read_all or seek_to == 'start': + self.logger.debug("Seeking to start") os.lseek(self.fd, 0, os.SEEK_SET) if not self.read_all or seek_to == 'end': + self.logger.debug("Seeking to end") os.lseek(self.fd, 0, os.SEEK_END) file_event_handler = EventHandler(callbacks={ @@ -258,43 +281,58 @@ def open(self, event=None, seek_to=None): 'moved': self.reopen_and_read, }) + self.logger.debug(f"Scheduling watch on file: '{self.path}'") self.watch = self.observer.schedule(file_event_handler, self.path) + self.logger.debug(f"Scheduled watch on file: '{self.path}'") # Avoid watching this twice + self.logger.debug(f"Parent watch: {self.parent_watch}") if not self.parent_watch: dir_event_handler = EventHandler(callbacks={ 'created': self.open_and_read, 'moved': self.reopen_and_read, }) + self.logger.debug(f"Scheduling watch on parent directory: '{self.parent_dir}'") self.parent_watch = self.observer.schedule(dir_event_handler, self.parent_dir) + self.logger.debug(f"Scheduled watch on parent directory: '{self.parent_dir}'") def close(self, event=None, emit_remaining=True): + self.logger.debug(f"Closing single file tail on '{self.path}'") # Reset the guard if self.buffer and emit_remaining: + self.logger.debug(f"Emitting remaining partial line: '{self.buffer}'") self.handler(self.path, self.buffer) self.buffer = '' if self.fd: os.close(self.fd) self.fd = None if self.watch: + self.logger.debug(f"Unscheduling file watch: {self._path}") self.observer.unschedule(self.watch) self.watch = None class TailManager(object): - def __init__(self, *args, **kwargs): + def __init__(self, *args, logger=None, **kwargs): + if logger is None: + raise Exception("TailManager was initialized without a logger") + + self.logger = logger self.tails = {} self.observer = Observer() def tail_file(self, path, handler, follow=False, read_all=False): if handler not in self.tails.setdefault(path, {}): + self.logger.debug(f"Tailing single file: {path}") sft = SingleFileTail(path, handler, follow=follow, read_all=read_all, - observer=self.observer) + observer=self.observer, + logger=self.logger) self.tails[path][handler] = sft def stop_tailing_file(self, path, handler): + self.logger.debug(f"Stopping tail on {path}") tailed_file = self.tails.get(path, {}).pop(handler) tailed_file.close() # Amortize some cleanup while we're at it @@ -302,6 +340,7 @@ def stop_tailing_file(self, path, handler): self.tails.pop(path) def run(self): + self.logger.debug("Running TailManager") self.start() try: while True: @@ -310,9 +349,11 @@ def run(self): self.stop() def start(self): + self.logger.debug("Starting TailManager") self.observer.start() def stop(self): + self.logger.debug("Stopping TailManager") for handlers in self.tails.values(): for tailed_file in handlers.values(): tailed_file.close() @@ -321,21 +362,23 @@ def stop(self): class FileWatchSensor(Sensor): - def __init__(self, *args, **kwargs): + def __init__(self, *args, logger=None, **kwargs): super().__init__(*args, **kwargs) self._stop = False self.trigger = None - self.logger = self.sensor_service.get_logger(__name__) + self.logger = logger or self.sensor_service.get_logger(__name__) def setup(self): - self.tail_manager = TailManager() + self.tail_manager = TailManager(logger=self.logger) def run(self): self.tail_manager.run() while not self._stop: + self.logger.debug("Sleeping for 60") eventlet.sleep(60) def cleanup(self): + self.logger.debug("Cleaning up FileWatchSensor") self._stop = True self.tail_manager.stop() @@ -381,7 +424,8 @@ def _handle_line(self, file_path, line): if __name__ == '__main__': - tm = TailManager() + logger = logging.getLogger(__name__) + tm = TailManager(logger=logger) tm.tail_file(__file__, handler=print) tm.run() diff --git a/contrib/linux/tests/test_file_watch_sensor.py b/contrib/linux/tests/test_file_watch_sensor.py index e6767459be..c968988cf2 100644 --- a/contrib/linux/tests/test_file_watch_sensor.py +++ b/contrib/linux/tests/test_file_watch_sensor.py @@ -15,6 +15,7 @@ # limitations under the License. import functools +import logging import os import pathlib import time @@ -23,6 +24,8 @@ WAIT_TIME = 0.1 +logger = logging.getLogger(__name__) + def test_single_file_tail_read_chunk_over_multibyte_character_boundary(): wide_characters = [None, None, '\u0130', '\u2050', '\U00088080'] @@ -51,7 +54,7 @@ def _run_n_byte_character_tests(chunk_size, n, length, char): fd = os.open(filename, os.O_RDONLY) - sft = SingleFileTail(None, None, fd=fd) + sft = SingleFileTail(None, None, fd=fd, logger=logger) result = sft.read_chunk(fd, chunk_size=chunk_size) @@ -80,7 +83,7 @@ def test_single_file_tail_read_chunk_with_bad_utf8_character(): fd = os.open(filename, os.O_RDONLY) - sft = SingleFileTail(None, None, fd=fd) + sft = SingleFileTail(None, None, fd=fd, logger=logger) err = None try: @@ -95,6 +98,18 @@ def test_single_file_tail_read_chunk_with_bad_utf8_character(): os.unlink(filename) +def test_single_file_tail_initialize_without_logger(): + try: + SingleFileTail(None, None, fd=None) + except Exception as e: + expected_message = "SingleFileTail was initialized without a logger" + if hasattr(e, 'message') and e.message != expected_message: + raise e + else: + raise AssertionError("SingleFileTail initialized fine without a " + "logger parameter") + + def test_single_file_tail_append_to_watched_file_with_absolute_path(): tailed_filename = (pathlib.Path.cwd() / pathlib.Path('tailed_file.txt')).resolve() with open(tailed_filename, 'w+') as f: @@ -108,7 +123,7 @@ def append_to_list(list_to_append, path, element): append_to_list_partial = functools.partial(append_to_list, appended_lines) - tm = TailManager() + tm = TailManager(logger=logger) tm.tail_file(tailed_filename, handler=append_to_list_partial) tm.start() @@ -142,7 +157,7 @@ def append_to_list(list_to_append, path, element): append_to_list_partial = functools.partial(append_to_list, appended_lines) - tm = TailManager() + tm = TailManager(logger=logger) tm.tail_file(tailed_filename, handler=append_to_list_partial) tm.start() @@ -195,7 +210,7 @@ def append_to_list(list_to_append, path, element): append_to_list_partial = functools.partial(append_to_list, appended_lines) - tm = TailManager() + tm = TailManager(logger=logger) tm.tail_file(tailed_filename, handler=append_to_list_partial) tm.start() time.sleep(WAIT_TIME) @@ -238,7 +253,7 @@ def append_to_list(list_to_append, path, element): append_to_list_partial = functools.partial(append_to_list, appended_lines) - tm = TailManager() + tm = TailManager(logger=logger) tm.tail_file(tailed_filename, handler=append_to_list_partial, follow=True) tm.start() @@ -312,7 +327,7 @@ def append_to_list(list_to_append, path, element): append_to_list_partial = functools.partial(append_to_list, appended_lines) - tm = TailManager() + tm = TailManager(logger=logger) tm.tail_file(tailed_filename, handler=append_to_list_partial, follow=False) tm.start() @@ -395,7 +410,7 @@ def append_to_list(list_to_append, path, element): append_to_list_partial = functools.partial(append_to_list, appended_lines) - tm = TailManager() + tm = TailManager(logger=logger) tm.tail_file(tailed_filename, handler=append_to_list_partial) tm.start() @@ -435,7 +450,7 @@ def append_to_list(list_to_append, path, element): append_to_list_partial = functools.partial(append_to_list, appended_lines) - tm = TailManager() + tm = TailManager(logger=logger) tm.tail_file(tailed_filename, handler=append_to_list_partial) tm.start() @@ -444,6 +459,18 @@ def append_to_list(list_to_append, path, element): tm.stop() +def test_tail_manager_initialized_without_logger(): + try: + TailManager() + except Exception as e: + expected_message = "TailManager was initialized without a logger" + if hasattr(e, 'message') and e.message != expected_message: + raise e + else: + raise AssertionError("TailManager initialized fine without a " + "logger parameter") + + def test_tail_manager_append_to_watched_file(): tailed_filename = 'tailed_file.txt' with open(tailed_filename, 'w+') as f: @@ -457,7 +484,7 @@ def append_to_list(list_to_append, path, element): append_to_list_partial = functools.partial(append_to_list, appended_lines) - tm = TailManager() + tm = TailManager(logger=logger) tm.tail_file(tailed_filename, handler=append_to_list_partial) tm.start() @@ -541,7 +568,7 @@ def append_to_list(list_to_append, path, element): append_to_list_partial = functools.partial(append_to_list, appended_lines) - tm = TailManager() + tm = TailManager(logger=logger) tm.tail_file(tailed_filename, handler=append_to_list_partial) tm.tail_file(tailed_filename, handler=append_to_list_partial) tm.start() @@ -626,7 +653,7 @@ def append_to_list(list_to_append, path, element): append_to_list_partial = functools.partial(append_to_list, appended_lines) - tm = TailManager() + tm = TailManager(logger=logger) tm.tail_file(tailed_filename, handler=append_to_list_partial) tm.start() @@ -666,7 +693,7 @@ def append_to_list(list_to_append, path, element): append_to_list_partial = functools.partial(append_to_list, appended_lines) - tm = TailManager() + tm = TailManager(logger=logger) tm.tail_file(tailed_filename, handler=append_to_list_partial) tm.start() From 490595d9784828ccab68eece645447ee2aedfa40 Mon Sep 17 00:00:00 2001 From: blag Date: Sun, 17 Jan 2021 01:18:52 -0700 Subject: [PATCH 11/35] No need for eventlet.sleep() when time.sleep() should do just fine --- contrib/linux/sensors/file_watch_sensor.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/contrib/linux/sensors/file_watch_sensor.py b/contrib/linux/sensors/file_watch_sensor.py index 8151010634..64ad2160d3 100644 --- a/contrib/linux/sensors/file_watch_sensor.py +++ b/contrib/linux/sensors/file_watch_sensor.py @@ -20,7 +20,6 @@ import time import sys -import eventlet from watchdog.observers import Observer from watchdog.events import FileSystemEventHandler @@ -375,7 +374,7 @@ def run(self): self.tail_manager.run() while not self._stop: self.logger.debug("Sleeping for 60") - eventlet.sleep(60) + time.sleep(60) def cleanup(self): self.logger.debug("Cleaning up FileWatchSensor") From 05603f6546bdd87b9569d9d77032180bfba5daee Mon Sep 17 00:00:00 2001 From: blag Date: Wed, 20 Jan 2021 13:23:41 -0700 Subject: [PATCH 12/35] Refactor TailManageer.run() to use time.sleep() --- contrib/linux/sensors/file_watch_sensor.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/contrib/linux/sensors/file_watch_sensor.py b/contrib/linux/sensors/file_watch_sensor.py index 64ad2160d3..e1570beabe 100644 --- a/contrib/linux/sensors/file_watch_sensor.py +++ b/contrib/linux/sensors/file_watch_sensor.py @@ -26,7 +26,10 @@ try: from st2reactor.sensor.base import Sensor except ImportError: - Sensor = object + class Sensor: + def __init__(self, *args, sensor_service=None, config=None, **kwargs): + self.sensor_service = sensor_service + self.config = config class EventHandler(FileSystemEventHandler): @@ -340,12 +343,8 @@ def stop_tailing_file(self, path, handler): def run(self): self.logger.debug("Running TailManager") - self.start() - try: - while True: - time.sleep(1) - finally: - self.stop() + while True: + time.sleep(1) def start(self): self.logger.debug("Starting TailManager") From 8a05c360ec6f85436d6b9c1fbc27e533e4724709 Mon Sep 17 00:00:00 2001 From: blag Date: Wed, 20 Jan 2021 13:19:55 -0700 Subject: [PATCH 13/35] Refactor calculating parent_dir --- contrib/linux/sensors/file_watch_sensor.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/contrib/linux/sensors/file_watch_sensor.py b/contrib/linux/sensors/file_watch_sensor.py index e1570beabe..f1793732e5 100644 --- a/contrib/linux/sensors/file_watch_sensor.py +++ b/contrib/linux/sensors/file_watch_sensor.py @@ -125,19 +125,13 @@ def get_path(self): def set_path(self, new_path): self.logger.debug(f"Setting path to {new_path}") self._path = pathlib.Path(new_path) - self.parent_dir = self.get_parent_path(self._path) - self.abs_path = (self.parent_dir / self._path).resolve() + self.abs_path = self._path.absolute().resolve() + self.parent_dir = self.abs_path.parent path = property(get_path, set_path) - def get_parent_path(self, path): - if path.is_absolute(): - return pathlib.Path(path).parent - else: - return (pathlib.Path.cwd() / path).parent - def get_event_src_path(self, event): - return (pathlib.Path.cwd() / pathlib.Path(event.src_path)).resolve() + return pathlib.Path(event.src_path).absolute().resolve() def read_chunk(self, fd, chunk_size=1024): self.logger.debug("Reading chunk") From 847d830fbbd958d8b5e3c3caf5d66dac3c1a393e Mon Sep 17 00:00:00 2001 From: blag Date: Wed, 20 Jan 2021 13:21:48 -0700 Subject: [PATCH 14/35] Refactor SingleFileTail.close() to keep parent watch open --- contrib/linux/sensors/file_watch_sensor.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/contrib/linux/sensors/file_watch_sensor.py b/contrib/linux/sensors/file_watch_sensor.py index f1793732e5..701cdd4269 100644 --- a/contrib/linux/sensors/file_watch_sensor.py +++ b/contrib/linux/sensors/file_watch_sensor.py @@ -293,20 +293,31 @@ def open(self, event=None, seek_to=None): self.parent_watch = self.observer.schedule(dir_event_handler, self.parent_dir) self.logger.debug(f"Scheduled watch on parent directory: '{self.parent_dir}'") - def close(self, event=None, emit_remaining=True): + def close(self, event=None, emit_remaining=True, end_parent_watch=True): self.logger.debug(f"Closing single file tail on '{self.path}'") # Reset the guard if self.buffer and emit_remaining: self.logger.debug(f"Emitting remaining partial line: '{self.buffer}'") self.handler(self.path, self.buffer) self.buffer = '' - if self.fd: - os.close(self.fd) - self.fd = None + if self.parent_watch and end_parent_watch: + self.logger.debug(f"Unscheduling parent directory watch: {self.parent_dir}") + self.observer.unschedule(self.parent_watch) + self.parent_watch = None + self.logger.debug(f"Unscheduled parent directory watch: {self.parent_dir}") if self.watch: self.logger.debug(f"Unscheduling file watch: {self._path}") self.observer.unschedule(self.watch) self.watch = None + self.logger.debug(f"Unscheduled file watch: {self._path}") + # Unscheduling a watch on a file descriptor requires a non-None fd, so + # we close the fd and set self.fd to None after unscheduling the file + # watch + if self.fd: + self.logger.debug(f"Closing file handle {self.fd}") + os.close(self.fd) + self.fd = None + self.logger.debug(f"Closed file handle") class TailManager(object): From 914b3f5809a4b9a6a2c2f80a497e3c40eea073e5 Mon Sep 17 00:00:00 2001 From: blag Date: Wed, 20 Jan 2021 13:20:40 -0700 Subject: [PATCH 15/35] Refactor follow logic --- contrib/linux/sensors/file_watch_sensor.py | 25 +++++++++++++--------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/contrib/linux/sensors/file_watch_sensor.py b/contrib/linux/sensors/file_watch_sensor.py index 701cdd4269..02962b54fb 100644 --- a/contrib/linux/sensors/file_watch_sensor.py +++ b/contrib/linux/sensors/file_watch_sensor.py @@ -215,20 +215,25 @@ def reopen_and_read(self, event=None, skip_to_end=False): self.logger.debug(f"Ignoring event for non-tracked file: '{event.src_path}'") return - # Save our current position into the file (this is a little wonky) - pos = os.lseek(self.fd, 0, os.SEEK_CUR) - self.logger.debug(f"Saving position ({pos}) into file {self.abs_path}") + # Guard against this being called twice - happens sometimes with inotify + if self.fd: + # Save our current position into the file (this is a little wonky) + pos = os.lseek(self.fd, 0, os.SEEK_CUR) + self.logger.debug(f"Saving position ({pos}) into file {self.abs_path}") # The file was moved and not recreated - # If we're following the file, don't emit the remainder of the last - # line - emit = not self.follow - self.close(event=event, emit_remaining=emit) - - # If we aren't following then don't reopen the file - # When the file is created again that will be handled by open_and_read if not self.follow: + # If we aren't following then don't reopen the file + # When the file is created again that will be handled by + # open_and_read + # But we do make sure to keep the parent file watch around to + # listen to created events + self.close(event=event, emit_remaining=True, end_parent_watch=False) return + else: + # If we are following the file, don't emit the remainder of the + # last line + self.close(event=event, emit_remaining=False) # Use the file's new location self.path = event.dest_path From 5caa4594227fb0e5717cfae80d366d9a1cc2d43d Mon Sep 17 00:00:00 2001 From: blag Date: Wed, 20 Jan 2021 13:22:59 -0700 Subject: [PATCH 16/35] Refactor TailManager.start() to make it idempotent --- contrib/linux/sensors/file_watch_sensor.py | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/contrib/linux/sensors/file_watch_sensor.py b/contrib/linux/sensors/file_watch_sensor.py index 02962b54fb..90313730c7 100644 --- a/contrib/linux/sensors/file_watch_sensor.py +++ b/contrib/linux/sensors/file_watch_sensor.py @@ -331,6 +331,7 @@ def __init__(self, *args, logger=None, **kwargs): raise Exception("TailManager was initialized without a logger") self.logger = logger + self.started = False self.tails = {} self.observer = Observer() @@ -357,16 +358,21 @@ def run(self): time.sleep(1) def start(self): - self.logger.debug("Starting TailManager") - self.observer.start() + if self.tails and not self.started: + self.logger.debug("Starting TailManager") + self.observer.start() + self.logger.debug(f"Started Observer, emitters: {self.observer.emitters}") + self.started = True def stop(self): - self.logger.debug("Stopping TailManager") - for handlers in self.tails.values(): - for tailed_file in handlers.values(): - tailed_file.close() - self.observer.stop() - self.observer.join() + if self.started: + self.logger.debug("Stopping TailManager") + for handlers in self.tails.values(): + for tailed_file in handlers.values(): + tailed_file.close() + self.observer.stop() + self.observer.join() + self.started = False class FileWatchSensor(Sensor): From e6d3911d7b319faf60ebec851b8352cc5dfa91b5 Mon Sep 17 00:00:00 2001 From: blag Date: Wed, 20 Jan 2021 13:24:16 -0700 Subject: [PATCH 17/35] Refactor FileWatchSensor.run() to use TailManager.run() --- contrib/linux/sensors/file_watch_sensor.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/contrib/linux/sensors/file_watch_sensor.py b/contrib/linux/sensors/file_watch_sensor.py index 90313730c7..9f3963a037 100644 --- a/contrib/linux/sensors/file_watch_sensor.py +++ b/contrib/linux/sensors/file_watch_sensor.py @@ -378,22 +378,17 @@ def stop(self): class FileWatchSensor(Sensor): def __init__(self, *args, logger=None, **kwargs): super().__init__(*args, **kwargs) - self._stop = False self.trigger = None self.logger = logger or self.sensor_service.get_logger(__name__) def setup(self): self.tail_manager = TailManager(logger=self.logger) + self.tail_manager.start() def run(self): self.tail_manager.run() - while not self._stop: - self.logger.debug("Sleeping for 60") - time.sleep(60) def cleanup(self): - self.logger.debug("Cleaning up FileWatchSensor") - self._stop = True self.tail_manager.stop() def add_trigger(self, trigger): @@ -411,6 +406,8 @@ def add_trigger(self, trigger): self.tail_manager.tail_file(file_path, self._handle_line) self.logger.info('Added file "%s"' % (file_path)) + self.tail_manager.start() + def update_trigger(self, trigger): pass @@ -441,7 +438,7 @@ def _handle_line(self, file_path, line): logger = logging.getLogger(__name__) tm = TailManager(logger=logger) tm.tail_file(__file__, handler=print) - tm.run() + tm.start() def halt(sig, frame): tm.stop() From 456dabaa86c0a122c01ecba816bc9737310392b6 Mon Sep 17 00:00:00 2001 From: blag Date: Wed, 20 Jan 2021 13:24:48 -0700 Subject: [PATCH 18/35] Break out tests into multiple modules --- contrib/linux/tests/test_file_watch_sensor.py | 759 +++--------------- contrib/linux/tests/test_single_file_tail.py | 603 ++++++++++++++ contrib/linux/tests/test_tail_manager.py | 647 +++++++++++++++ 3 files changed, 1371 insertions(+), 638 deletions(-) create mode 100644 contrib/linux/tests/test_single_file_tail.py create mode 100644 contrib/linux/tests/test_tail_manager.py diff --git a/contrib/linux/tests/test_file_watch_sensor.py b/contrib/linux/tests/test_file_watch_sensor.py index c968988cf2..c42a110306 100644 --- a/contrib/linux/tests/test_file_watch_sensor.py +++ b/contrib/linux/tests/test_file_watch_sensor.py @@ -18,708 +18,191 @@ import logging import os import pathlib +import sys +import threading import time -from file_watch_sensor import SingleFileTail, TailManager +import eventlet +import mock +from file_watch_sensor import FileWatchSensor -WAIT_TIME = 0.1 +WAIT_TIME = 1 logger = logging.getLogger(__name__) -def test_single_file_tail_read_chunk_over_multibyte_character_boundary(): - wide_characters = [None, None, '\u0130', '\u2050', '\U00088080'] - for n in range(2, 5): - yield from _gen_n_byte_character_tests(1024, n, wide_characters[n]) +def test_file_watch_sensor(): + mock_sensor_service = mock.MagicMock() + mock_logger = mock.MagicMock() - for n in range(2, 5): - yield from _gen_n_byte_character_tests(2048, n, wide_characters[n]) + filename = 'test.txt' + filepath = pathlib.Path(filename).absolute().resolve() + filepath.touch() - for n in range(2, 5): - yield from _gen_n_byte_character_tests(4096, n, wide_characters[n]) + fws = FileWatchSensor(sensor_service=mock_sensor_service, config={}, + logger=mock_logger) - -def _gen_n_byte_character_tests(chunk_size, n, char): - for length in range(chunk_size, chunk_size + n + 1): - yield _run_n_byte_character_tests, chunk_size, n, length, char - - -def _run_n_byte_character_tests(chunk_size, n, length, char): - filename = f'chunk_boundary_{n}u_{length}.txt' - - with open(filename, 'wb+') as f: - # Write out a file that is of the given length - # aaaaaa...aaa\x82 - f.write(('a' * (length - n) + char).encode('utf-8')) - - fd = os.open(filename, os.O_RDONLY) - - sft = SingleFileTail(None, None, fd=fd, logger=logger) - - result = sft.read_chunk(fd, chunk_size=chunk_size) - - os.close(fd) - os.unlink(filename) - - if length < chunk_size + n: - assert result == ('a' * (length - n) + char) - else: - assert result == ('a' * chunk_size) - - -def test_single_file_tail_read_chunk_with_bad_utf8_character(): - filename = 'bad_utf8_character.txt' - - utf8_str = '\U00088080' - utf8_bytes = utf8_str.encode('utf-8') - chopped_utf8_bytes = utf8_bytes[:-1] - - with open(filename, 'wb+') as f: - # Write out a file that is of the given length - # aaaaaa...aaa\x82 - f.write(b'a') - f.write(chopped_utf8_bytes) - f.write(b'a') - - fd = os.open(filename, os.O_RDONLY) - - sft = SingleFileTail(None, None, fd=fd, logger=logger) - - err = None - try: - sft.read_chunk(fd) - except Exception as e: - err = e - finally: - assert err is not None - assert isinstance(err, UnicodeDecodeError) - - os.close(fd) - os.unlink(filename) - - -def test_single_file_tail_initialize_without_logger(): - try: - SingleFileTail(None, None, fd=None) - except Exception as e: - expected_message = "SingleFileTail was initialized without a logger" - if hasattr(e, 'message') and e.message != expected_message: - raise e - else: - raise AssertionError("SingleFileTail initialized fine without a " - "logger parameter") - - -def test_single_file_tail_append_to_watched_file_with_absolute_path(): - tailed_filename = (pathlib.Path.cwd() / pathlib.Path('tailed_file.txt')).resolve() - with open(tailed_filename, 'w+') as f: - f.write("Preexisting text line 1\n") - f.write("Preexisting text line 2\n") - - appended_lines = [] - - def append_to_list(list_to_append, path, element): - list_to_append.append(element) - - append_to_list_partial = functools.partial(append_to_list, appended_lines) - - tm = TailManager(logger=logger) - tm.tail_file(tailed_filename, handler=append_to_list_partial) - tm.start() - - with open(tailed_filename, 'a+') as f: - f.write("Added line 1\n") - time.sleep(WAIT_TIME) - - assert appended_lines == [ - "Added line 1", - ] - - tm.stop_tailing_file(tailed_filename, handler=append_to_list_partial) - - tm.stop() - - os.unlink(tailed_filename) - - -def test_single_file_tail_not_watched_file(): - tailed_filename = 'tailed_file.txt' - not_tailed_filename = 'not_tailed_file.txt' - new_not_tailed_filename = not_tailed_filename.replace('.txt', '_moved.txt') - with open(tailed_filename, 'w+') as f: - f.write("Preexisting text line 1\n") - f.write("Preexisting text line 2\n") - - appended_lines = [] - - def append_to_list(list_to_append, path, element): - list_to_append.append(element) - - append_to_list_partial = functools.partial(append_to_list, appended_lines) - - tm = TailManager(logger=logger) - tm.tail_file(tailed_filename, handler=append_to_list_partial) - tm.start() - - with open(tailed_filename, 'a+') as f: - f.write("Added line 1\n") time.sleep(WAIT_TIME) - assert appended_lines == [ - "Added line 1", - ] + fws.setup() - with open(not_tailed_filename, 'a+') as f: - f.write("Added line 1 - not tailed\n") time.sleep(WAIT_TIME) - assert appended_lines == [ - "Added line 1", - ] + # th = threading.Thread(target=fws.run) + th = eventlet.spawn(fws.run) + # th.start() - os.replace(not_tailed_filename, new_not_tailed_filename) time.sleep(WAIT_TIME) - assert appended_lines == [ - "Added line 1", - ] - - tm.stop_tailing_file(tailed_filename, handler=append_to_list_partial) + fws.add_trigger({ + 'id': 'asdf.adsfasdf-asdf-asdf-asdfasdfasdf', + 'pack': 'linux', + 'name': 'asdf.adsfasdf-asdf-asdf-asdfasdfasdf', + 'ref': 'linux.asdf.adsfasdf-asdf-asdf-asdfasdfasdf', + 'uid': 'trigger:linux:asdf.adsfasdf-asdf-asdf-asdfasdfasdf', + 'type': 'linux.file_watch.line', + 'parameters': { + 'file_path': filepath, + 'follow': True, + }, + }) - tm.stop() time.sleep(WAIT_TIME) - assert appended_lines == [ - "Added line 1", - ] - - os.unlink(tailed_filename) - os.unlink(new_not_tailed_filename) - - -def test_single_file_tail_watch_nonexistent_file(): - tailed_filename = 'tailed_file.txt' - - if os.path.exists(tailed_filename): - os.unlink(tailed_filename) - - appended_lines = [] - - def append_to_list(list_to_append, path, element): - list_to_append.append(element) - - append_to_list_partial = functools.partial(append_to_list, appended_lines) - - tm = TailManager(logger=logger) - tm.tail_file(tailed_filename, handler=append_to_list_partial) - tm.start() - time.sleep(WAIT_TIME) - - assert appended_lines == [] - - with open(tailed_filename, 'w+') as f: - f.write("Added line 1\n") - time.sleep(WAIT_TIME) - - assert appended_lines == [ - "Added line 1", - ] - - tm.stop_tailing_file(tailed_filename, handler=append_to_list_partial) - - tm.stop() - time.sleep(WAIT_TIME) - - os.unlink(tailed_filename) - - -def test_single_file_tail_follow_watched_file_moved(): - tailed_filename = 'tailed_file_to_move.txt' - new_filename = tailed_filename.replace('_to_move.txt', '_moved.txt') - - if os.path.exists(new_filename): - os.unlink(new_filename) - if os.path.exists(tailed_filename): - os.unlink(tailed_filename) - - with open(tailed_filename, 'w+') as f: - f.write("Preexisting text line 1\n") - f.write("Preexisting text line 2\n") - - appended_lines = [] - - def append_to_list(list_to_append, path, element): - list_to_append.append(element) - - append_to_list_partial = functools.partial(append_to_list, appended_lines) - - tm = TailManager(logger=logger) - tm.tail_file(tailed_filename, handler=append_to_list_partial, follow=True) - tm.start() - - with open(tailed_filename, 'a+') as f: + with open(filepath, 'a') as f: f.write("Added line 1\n") - time.sleep(WAIT_TIME) - - assert appended_lines == [ - "Added line 1", - ] - - with open(tailed_filename, 'a+') as f: - f.write("Added line 2") # No newline - time.sleep(WAIT_TIME) - - assert appended_lines == [ - "Added line 1", - ] - - os.replace(tailed_filename, new_filename) - time.sleep(WAIT_TIME) - assert appended_lines == [ - "Added line 1", - ] - - with open(new_filename, 'a+') as f: - f.write(" - end of line 2\n") time.sleep(WAIT_TIME) - assert appended_lines == [ - "Added line 1", - "Added line 2 - end of line 2", - ] + with open(filepath, 'a') as f: + f.write("Added line 2\n") - with open(tailed_filename, 'w+') as f: - f.write("New file - text line 1\n") - f.write("New file - text line 2\n") time.sleep(WAIT_TIME) - assert appended_lines == [ - "Added line 1", - "Added line 2 - end of line 2", + expected_calls = [ + mock.call( + trigger='linux.asdf.adsfasdf-asdf-asdf-asdfasdfasdf', + payload={ + 'file_path': pathlib.PosixPath('/vagrant/contrib/linux/test.txt'), + 'file_name': 'test.txt', + 'line': 'Added line 1', + }, + ), + mock.call( + trigger='linux.asdf.adsfasdf-asdf-asdf-asdfasdfasdf', + payload={ + 'file_path': pathlib.PosixPath('/vagrant/contrib/linux/test.txt'), + 'file_name': 'test.txt', + 'line': 'Added line 2', + }, + ) ] + mock_sensor_service.dispatch.assert_has_calls(expected_calls, any_order=False) + print(mock_logger.method_calls) + # th.join() - tm.stop_tailing_file(tailed_filename, handler=append_to_list_partial) - - tm.stop() - - os.unlink(new_filename) - os.unlink(tailed_filename) - - -def test_single_file_tail_not_followed_watched_file_moved(): - tailed_filename = 'tailed_file_to_move.txt' - new_filename = tailed_filename.replace('_to_move.txt', '_moved.txt') + fws.cleanup() - if os.path.exists(new_filename): - os.unlink(new_filename) - if os.path.exists(tailed_filename): - os.unlink(tailed_filename) + os.unlink(filepath) - with open(tailed_filename, 'w+') as f: - f.write("Preexisting text line 1\n") - f.write("Preexisting text line 2\n") - - appended_lines = [] - - def append_to_list(list_to_append, path, element): - list_to_append.append(element) - - append_to_list_partial = functools.partial(append_to_list, appended_lines) - - tm = TailManager(logger=logger) - tm.tail_file(tailed_filename, handler=append_to_list_partial, follow=False) - tm.start() - - with open(tailed_filename, 'a+') as f: - f.write("Added line 1\n") - time.sleep(WAIT_TIME) - assert appended_lines == [ - "Added line 1", - ] +def test_file_watch_sensor_without_trigger_filepath(): + mock_sensor_service = mock.MagicMock() + mock_logger = mock.MagicMock() - with open(tailed_filename, 'a+') as f: - f.write("Added line 2") # No newline - time.sleep(WAIT_TIME) + filename = 'test.txt' + filepath = pathlib.Path(filename).absolute().resolve() + filepath.touch() - assert appended_lines == [ - "Added line 1", - ] + fws = FileWatchSensor(sensor_service=mock_sensor_service, config={}, + logger=mock_logger) - os.replace(tailed_filename, new_filename) time.sleep(WAIT_TIME) - assert appended_lines == [ - "Added line 1", - "Added line 2", - ] + fws.setup() - with open(new_filename, 'a+') as f: - f.write(" - end of line 2\n") time.sleep(WAIT_TIME) - assert appended_lines == [ - "Added line 1", - "Added line 2", - ] + # th = threading.Thread(target=fws.run) + th = eventlet.spawn(fws.run) + # th.start() - with open(tailed_filename, 'w+') as f: - f.write("Recreated file - text line 1\n") - f.write("Recreated file - text line 2\n") time.sleep(WAIT_TIME) - assert appended_lines == [ - "Added line 1", - "Added line 2", - "Recreated file - text line 1", - "Recreated file - text line 2", - ] - - tm.stop_tailing_file(tailed_filename, handler=append_to_list_partial) - - tm.stop() + result = fws.add_trigger({ + 'id': 'asdf.adsfasdf-asdf-asdf-asdfasdfasdf', + 'pack': 'linux', + 'name': 'asdf.adsfasdf-asdf-asdf-asdfasdfasdf', + 'ref': 'linux.asdf.adsfasdf-asdf-asdf-asdfasdfasdf', + 'uid': 'trigger:linux:asdf.adsfasdf-asdf-asdf-asdfasdfasdf', + 'type': 'linux.file_watch.line', + 'parameters': { + # 'file_path': filepath, + 'follow': True, + }, + }) - os.unlink(new_filename) - os.unlink(tailed_filename) + assert result is None -def test_single_file_tail_non_watched_file_moved(): - tailed_filename = 'tailed_file_to_move.txt' - not_tailed_filename = f'not_{tailed_filename}' - new_not_tailed_filename = not_tailed_filename.replace('_to_move.txt', '_moved.txt') +def test_file_watch_sensor_without_trigger_ref(): + mock_sensor_service = mock.MagicMock() + mock_logger = mock.MagicMock() - if os.path.exists(not_tailed_filename): - os.unlink(not_tailed_filename) - if os.path.exists(new_not_tailed_filename): - os.unlink(new_not_tailed_filename) - if os.path.exists(tailed_filename): - os.unlink(tailed_filename) + filename = 'test.txt' + filepath = pathlib.Path(filename).absolute().resolve() + filepath.touch() - with open(not_tailed_filename, 'w+') as f: - f.write("Text here will not be monitored\n") + fws = FileWatchSensor(sensor_service=mock_sensor_service, config={}, + logger=mock_logger) - with open(tailed_filename, 'w+') as f: - f.write("Preexisting text line 1\n") - f.write("Preexisting text line 2\n") - - appended_lines = [] - - def append_to_list(list_to_append, path, element): - list_to_append.append(element) - - append_to_list_partial = functools.partial(append_to_list, appended_lines) - - tm = TailManager(logger=logger) - tm.tail_file(tailed_filename, handler=append_to_list_partial) - tm.start() - - with open(tailed_filename, 'a+') as f: - f.write("Added line 1\n") time.sleep(WAIT_TIME) - assert appended_lines == [ - "Added line 1", - ] + fws.setup() - os.replace(not_tailed_filename, new_not_tailed_filename) time.sleep(WAIT_TIME) - assert appended_lines == [ - "Added line 1", - ] - - tm.stop_tailing_file(tailed_filename, handler=append_to_list_partial) - - tm.stop() - - os.unlink(new_not_tailed_filename) - os.unlink(tailed_filename) - - -def test_single_file_tail_watched_file_deleted(): - tailed_filename = 'tailed_file_deleted.txt' - with open(tailed_filename, 'w+') as f: - f.write("Preexisting text line 1\n") - f.write("Preexisting text line 2\n") - - appended_lines = [] - - def append_to_list(list_to_append, path, element): - list_to_append.append(element) - - append_to_list_partial = functools.partial(append_to_list, appended_lines) - - tm = TailManager(logger=logger) - tm.tail_file(tailed_filename, handler=append_to_list_partial) - tm.start() - - os.unlink(tailed_filename) - - tm.stop() + # th = threading.Thread(target=fws.run) + th = eventlet.spawn(fws.run) + # th.start() + time.sleep(WAIT_TIME) -def test_tail_manager_initialized_without_logger(): try: - TailManager() + fws.add_trigger({ + 'id': 'asdf.adsfasdf-asdf-asdf-asdfasdfasdf', + 'pack': 'linux', + 'name': 'asdf.adsfasdf-asdf-asdf-asdfasdfasdf', + # 'ref': 'linux.asdf.adsfasdf-asdf-asdf-asdfasdfasdf', + 'uid': 'trigger:linux:asdf.adsfasdf-asdf-asdf-asdfasdfasdf', + 'type': 'linux.file_watch.line', + 'parameters': { + 'file_path': filepath, + 'follow': True, + }, + }) except Exception as e: - expected_message = "TailManager was initialized without a logger" - if hasattr(e, 'message') and e.message != expected_message: + # Make sure we ignore the right exception + if 'did not contain a ref' not in str(e): raise e else: - raise AssertionError("TailManager initialized fine without a " - "logger parameter") - - -def test_tail_manager_append_to_watched_file(): - tailed_filename = 'tailed_file.txt' - with open(tailed_filename, 'w+') as f: - f.write("Preexisting text line 1\n") - f.write("Preexisting text line 2\n") - - appended_lines = [] - - def append_to_list(list_to_append, path, element): - list_to_append.append(element) - - append_to_list_partial = functools.partial(append_to_list, appended_lines) - - tm = TailManager(logger=logger) - tm.tail_file(tailed_filename, handler=append_to_list_partial) - tm.start() - - with open(tailed_filename, 'a+') as f: - f.write("Added line 1\n") - time.sleep(WAIT_TIME) - - assert appended_lines == [ - "Added line 1", - ] - - with open(tailed_filename, 'a+') as f: - f.write("Added line 2\n") - time.sleep(WAIT_TIME) - - assert appended_lines == [ - "Added line 1", - "Added line 2", - ] - - with open(tailed_filename, 'a+') as f: - f.write("Start of added partial line 1") - time.sleep(WAIT_TIME) - - assert appended_lines == [ - "Added line 1", - "Added line 2", - ] - - with open(tailed_filename, 'a+') as f: - f.write(" - finished partial line 1\nStart of added partial line 2") - time.sleep(WAIT_TIME) - - assert appended_lines == [ - "Added line 1", - "Added line 2", - "Start of added partial line 1 - finished partial line 1", - ] - - with open(tailed_filename, 'a+') as f: - f.write(" - finished partial line 2\n") - time.sleep(WAIT_TIME) - - assert appended_lines == [ - "Added line 1", - "Added line 2", - "Start of added partial line 1 - finished partial line 1", - "Start of added partial line 2 - finished partial line 2", - ] - - with open(tailed_filename, 'a+') as f: - f.write("Final line without a newline") - time.sleep(WAIT_TIME) - - tm.stop_tailing_file(tailed_filename, handler=append_to_list_partial) - - tm.stop() - - time.sleep(WAIT_TIME) - assert appended_lines == [ - "Added line 1", - "Added line 2", - "Start of added partial line 1 - finished partial line 1", - "Start of added partial line 2 - finished partial line 2", - "Final line without a newline", - ] - - os.unlink(tailed_filename) - - -def test_tail_manager_tail_file_twice(): - tailed_filename = 'tailed_file.txt' - with open(tailed_filename, 'w+') as f: - f.write("Preexisting text line 1\n") - f.write("Preexisting text line 2\n") - - appended_lines = [] - - def append_to_list(list_to_append, path, element): - list_to_append.append(element) - - append_to_list_partial = functools.partial(append_to_list, appended_lines) - - tm = TailManager(logger=logger) - tm.tail_file(tailed_filename, handler=append_to_list_partial) - tm.tail_file(tailed_filename, handler=append_to_list_partial) - tm.start() - - with open(tailed_filename, 'a+') as f: - f.write("Added line 1\n") - time.sleep(WAIT_TIME) - - assert appended_lines == [ - "Added line 1", - ] - - with open(tailed_filename, 'a+') as f: - f.write("Added line 2\n") - time.sleep(WAIT_TIME) - - assert appended_lines == [ - "Added line 1", - "Added line 2", - ] - - with open(tailed_filename, 'a+') as f: - f.write("Start of added partial line 1") - time.sleep(WAIT_TIME) - - assert appended_lines == [ - "Added line 1", - "Added line 2", - ] - - with open(tailed_filename, 'a+') as f: - f.write(" - finished partial line 1\nStart of added partial line 2") - time.sleep(WAIT_TIME) - - assert appended_lines == [ - "Added line 1", - "Added line 2", - "Start of added partial line 1 - finished partial line 1", - ] - - with open(tailed_filename, 'a+') as f: - f.write(" - finished partial line 2\n") - time.sleep(WAIT_TIME) - - assert appended_lines == [ - "Added line 1", - "Added line 2", - "Start of added partial line 1 - finished partial line 1", - "Start of added partial line 2 - finished partial line 2", - ] - - with open(tailed_filename, 'a+') as f: - f.write("Final line without a newline") - time.sleep(WAIT_TIME) - - tm.stop_tailing_file(tailed_filename, handler=append_to_list_partial) - - tm.stop() - - time.sleep(WAIT_TIME) - assert appended_lines == [ - "Added line 1", - "Added line 2", - "Start of added partial line 1 - finished partial line 1", - "Start of added partial line 2 - finished partial line 2", - "Final line without a newline", - ] - - os.unlink(tailed_filename) - - -def test_tail_manager_stop(): - tailed_filename = 'tailed_file_stop.txt' - with open(tailed_filename, 'w+') as f: - f.write("Preexisting text line 1\n") - f.write("Preexisting text line 2\n") - - appended_lines = [] - - def append_to_list(list_to_append, path, element): - list_to_append.append(element) - - append_to_list_partial = functools.partial(append_to_list, appended_lines) - - tm = TailManager(logger=logger) - tm.tail_file(tailed_filename, handler=append_to_list_partial) - tm.start() - - with open(tailed_filename, 'a+') as f: - f.write("Added line 1\n") - time.sleep(WAIT_TIME) - - assert appended_lines == [ - "Added line 1", - ] - - with open(tailed_filename, 'a+') as f: - f.write("Final line without a newline") - time.sleep(WAIT_TIME) - - tm.stop() - - time.sleep(WAIT_TIME) - assert appended_lines == [ - "Added line 1", - "Final line without a newline", - ] - - os.unlink(tailed_filename) - - -def test_tail_manager_stop_twice(): - tailed_filename = 'tailed_file_stop.txt' - with open(tailed_filename, 'w+') as f: - f.write("Preexisting text line 1\n") - f.write("Preexisting text line 2\n") - - appended_lines = [] - - def append_to_list(list_to_append, path, element): - list_to_append.append(element) - - append_to_list_partial = functools.partial(append_to_list, appended_lines) - - tm = TailManager(logger=logger) - tm.tail_file(tailed_filename, handler=append_to_list_partial) - tm.start() - - with open(tailed_filename, 'a+') as f: - f.write("Added line 1\n") - time.sleep(WAIT_TIME) - - assert appended_lines == [ - "Added line 1", - ] + raise AssertionError("FileWatchSensor.add_trigger() did not raise an " + "exception when passed a trigger without a ref") + finally: + os.unlink(filepath) - with open(tailed_filename, 'a+') as f: - f.write("Final line without a newline") - time.sleep(WAIT_TIME) - tm.stop() - tm.stop() +if __name__ == '__main__': + # logger.setLevel(logging.DEBUG) - time.sleep(WAIT_TIME) - assert appended_lines == [ - "Added line 1", - "Final line without a newline", - ] + # handler = logging.StreamHandler(sys.stderr) + # handler.setLevel(logging.DEBUG) + # formatter = logging.Formatter('%(name)s: %(levelname)s: %(message)s') - os.unlink(tailed_filename) + # logger.addHandler(handler) + # test_single_file_tail_watched_file_deleted() -if __name__ == '__main__': - test_single_file_tail_not_followed_watched_file_moved() + test_file_watch_sensor() + test_file_watch_sensor_without_trigger_filepath() + test_file_watch_sensor_without_trigger_ref() diff --git a/contrib/linux/tests/test_single_file_tail.py b/contrib/linux/tests/test_single_file_tail.py new file mode 100644 index 0000000000..18fb5f23a6 --- /dev/null +++ b/contrib/linux/tests/test_single_file_tail.py @@ -0,0 +1,603 @@ +#!/usr/bin/env python + +# Copyright 2020 The StackStorm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import functools +import logging +import os +import pathlib +import time + +from watchdog.observers import Observer + +from file_watch_sensor import SingleFileTail + +WAIT_TIME = 1 + +logger = logging.getLogger(__name__) + + +def test_read_chunk_over_multibyte_character_boundary(): + wide_characters = [None, None, '\u0130', '\u2050', '\U00088080'] + for n in range(2, 5): + yield from _gen_n_byte_character_tests(1024, n, wide_characters[n]) + + for n in range(2, 5): + yield from _gen_n_byte_character_tests(2048, n, wide_characters[n]) + + for n in range(2, 5): + yield from _gen_n_byte_character_tests(4096, n, wide_characters[n]) + + +def _gen_n_byte_character_tests(chunk_size, n, char): + for length in range(chunk_size, chunk_size + n + 1): + yield _run_n_byte_character_tests, chunk_size, n, length, char + + +def _run_n_byte_character_tests(chunk_size, n, length, char): + filename = f'chunk_boundary_{n}u_{length}.txt' + + with open(filename, 'wb+') as f: + # Write out a file that is of the given length + # aaaaaa...aaa\x82 + f.write(('a' * (length - n) + char).encode('utf-8')) + + fd = os.open(filename, os.O_RDONLY) + + sft = SingleFileTail(None, None, fd=fd, logger=logger) + + result = sft.read_chunk(fd, chunk_size=chunk_size) + + os.close(fd) + os.unlink(filename) + + if length < chunk_size + n: + assert result == ('a' * (length - n) + char) + else: + assert result == ('a' * chunk_size) + + +def test_read_chunk_with_bad_utf8_character(): + filename = 'bad_utf8_character.txt' + + utf8_str = '\U00088080' + utf8_bytes = utf8_str.encode('utf-8') + chopped_utf8_bytes = utf8_bytes[:-1] + + with open(filename, 'wb+') as f: + # Write out a file that is of the given length + # aaaaaa...aaa\x82 + f.write(b'a') + f.write(chopped_utf8_bytes) + f.write(b'a') + + fd = os.open(filename, os.O_RDONLY) + + sft = SingleFileTail(None, None, fd=fd, logger=logger) + + err = None + try: + sft.read_chunk(fd) + except Exception as e: + err = e + finally: + assert err is not None + assert isinstance(err, UnicodeDecodeError) + + os.close(fd) + os.unlink(filename) + + +def test_read_chunk_from_nonexistent_file(): + filename = 'nonexistent_file.txt' + + with open(filename, 'w+') as f: + f.write("This file will not exist in a few moments") + + fd = os.open(filename, os.O_RDONLY) + os.close(fd) + + os.unlink(filename) + + sft = SingleFileTail(None, None, fd=fd, logger=logger) + + assert sft.read_chunk(fd=fd) == '' + + +# Helper function + +def append_to_list(list_to_append, path, element): + logger.debug(f"Appending ({path}):\n{element} to {list_to_append}") + list_to_append.append(element) + + +def test_initialize_without_logger(): + try: + SingleFileTail(None, None, fd=None) + except Exception as e: + expected_message = "SingleFileTail was initialized without a logger" + if hasattr(e, 'message') and e.message != expected_message: + raise e + else: + raise AssertionError("SingleFileTail initialized fine without a " + "logger parameter") + + +def test_append_to_watched_file_with_absolute_path(): + tailed_filename = (pathlib.Path.cwd() / pathlib.Path('tailed_file.txt')).resolve() + with open(tailed_filename, 'w+') as f: + f.write("Preexisting text line 1\n") + f.write("Preexisting text line 2\n") + + appended_lines = [] + append_to_list_partial = functools.partial(append_to_list, appended_lines) + + observer = Observer() + + sft = SingleFileTail(tailed_filename, append_to_list_partial, + observer=observer, logger=logger) + + observer.start() + time.sleep(WAIT_TIME) + + with open(tailed_filename, 'a+') as f: + f.write("Added line 1\n") + time.sleep(WAIT_TIME) + + assert appended_lines == [ + "Added line 1", + ] + + sft.close() + + observer.stop() + + os.unlink(tailed_filename) + + +def test_append_to_watched_file_with_relative_path(): + tailed_filename = pathlib.Path('tailed_file.txt') + with open(tailed_filename, 'w+') as f: + f.write("Preexisting text line 1\n") + f.write("Preexisting text line 2\n") + + appended_lines = [] + append_to_list_partial = functools.partial(append_to_list, appended_lines) + + observer = Observer() + + sft = SingleFileTail(tailed_filename, append_to_list_partial, + observer=observer, logger=logger) + + observer.start() + time.sleep(WAIT_TIME) + + with open(tailed_filename, 'a+') as f: + f.write("Added line 1\n") + time.sleep(WAIT_TIME) + + assert appended_lines == [ + "Added line 1", + ] + + sft.close() + + observer.stop() + + os.unlink(tailed_filename) + + +def test_append_to_watched_file_observer_start_first(): + tailed_filename = pathlib.Path('tailed_file.txt') + with open(tailed_filename, 'w+') as f: + f.write("Preexisting text line 1\n") + f.write("Preexisting text line 2\n") + + appended_lines = [] + append_to_list_partial = functools.partial(append_to_list, appended_lines) + + observer = Observer() + observer.start() + + sft = SingleFileTail(tailed_filename, append_to_list_partial, + observer=observer, logger=logger) + + with open(tailed_filename, 'a+') as f: + f.write("Added line 1\n") + time.sleep(WAIT_TIME) + + assert appended_lines == [ + "Added line 1", + ] + + sft.close() + + observer.stop() + + os.unlink(tailed_filename) + + +def test_not_watched_file(): + tailed_filename = 'tailed_file.txt' + not_tailed_filename = 'not_tailed_file.txt' + new_not_tailed_filename = not_tailed_filename.replace('.txt', '_moved.txt') + with open(tailed_filename, 'w+') as f: + f.write("Preexisting text line 1\n") + f.write("Preexisting text line 2\n") + + appended_lines = [] + append_to_list_partial = functools.partial(append_to_list, appended_lines) + + observer = Observer() + + sft = SingleFileTail(tailed_filename, append_to_list_partial, + observer=observer, logger=logger) + + observer.start() + + with open(tailed_filename, 'a+') as f: + f.write("Added line 1\n") + time.sleep(WAIT_TIME) + + assert appended_lines == [ + "Added line 1", + ] + + with open(not_tailed_filename, 'a+') as f: + f.write("Added line 1 - not tailed\n") + time.sleep(WAIT_TIME) + + assert appended_lines == [ + "Added line 1", + ] + + os.replace(not_tailed_filename, new_not_tailed_filename) + time.sleep(WAIT_TIME) + + assert appended_lines == [ + "Added line 1", + ] + + sft.close() + time.sleep(WAIT_TIME) + + assert appended_lines == [ + "Added line 1", + ] + + observer.stop() + + os.unlink(tailed_filename) + os.unlink(new_not_tailed_filename) + + +def test_watch_nonexistent_file(): + tailed_filename = 'tailed_file.txt' + + if os.path.exists(tailed_filename): + os.unlink(tailed_filename) + + appended_lines = [] + append_to_list_partial = functools.partial(append_to_list, appended_lines) + + observer = Observer() + + sft = SingleFileTail(tailed_filename, append_to_list_partial, + observer=observer, logger=logger) + + observer.start() + + assert appended_lines == [] + + assert not os.path.exists(tailed_filename) + + with open(tailed_filename, 'w+') as f: + f.write("Added line 1\n") + time.sleep(WAIT_TIME) + + assert os.path.exists(tailed_filename) + + assert appended_lines == [ + "Added line 1", + ] + + sft.close() + time.sleep(WAIT_TIME) + + observer.stop() + + os.unlink(tailed_filename) + + +def test_follow_watched_file_moved(): + tailed_filename = 'tailed_file_to_move.txt' + new_filename = tailed_filename.replace('_to_move.txt', '_moved.txt') + + if os.path.exists(new_filename): + os.unlink(new_filename) + if os.path.exists(tailed_filename): + os.unlink(tailed_filename) + + with open(tailed_filename, 'w+') as f: + f.write("Preexisting text line 1\n") + f.write("Preexisting text line 2\n") + + appended_lines = [] + append_to_list_partial = functools.partial(append_to_list, appended_lines) + + observer = Observer() + + sft = SingleFileTail(tailed_filename, append_to_list_partial, follow=True, + observer=observer, logger=logger) + + observer.start() + + with open(tailed_filename, 'a+') as f: + f.write("Added line 1\n") + time.sleep(WAIT_TIME) + + assert appended_lines == [ + "Added line 1", + ] + + with open(tailed_filename, 'a+') as f: + f.write("Added line 2") # No newline + time.sleep(WAIT_TIME) + + assert appended_lines == [ + "Added line 1", + ] + + os.replace(tailed_filename, new_filename) + time.sleep(WAIT_TIME) + + assert appended_lines == [ + "Added line 1", + ] + + with open(new_filename, 'a+') as f: + f.write(" - end of line 2\n") + time.sleep(WAIT_TIME) + + assert appended_lines == [ + "Added line 1", + "Added line 2 - end of line 2", + ] + + with open(tailed_filename, 'w+') as f: + f.write("New file - text line 1\n") + f.write("New file - text line 2\n") + time.sleep(WAIT_TIME) + + assert appended_lines == [ + "Added line 1", + "Added line 2 - end of line 2", + ] + + sft.close() + + observer.stop() + + os.unlink(new_filename) + os.unlink(tailed_filename) + + +def test_not_followed_watched_file_moved(): + tailed_filename = 'tailed_file_to_move.txt' + new_filename = tailed_filename.replace('_to_move.txt', '_moved.txt') + + if os.path.exists(new_filename): + os.unlink(new_filename) + if os.path.exists(tailed_filename): + os.unlink(tailed_filename) + + with open(tailed_filename, 'w+') as f: + f.write("Preexisting text line 1\n") + f.write("Preexisting text line 2\n") + + appended_lines = [] + append_to_list_partial = functools.partial(append_to_list, appended_lines) + + observer = Observer() + + sft = SingleFileTail(tailed_filename, append_to_list_partial, follow=False, + observer=observer, logger=logger) + + observer.start() + + with open(tailed_filename, 'a+') as f: + f.write("Added line 1\n") + time.sleep(WAIT_TIME) + + assert appended_lines == [ + "Added line 1", + ] + + with open(tailed_filename, 'a+') as f: + f.write("Added line 2") # No newline + time.sleep(WAIT_TIME) + + assert appended_lines == [ + "Added line 1", + ] + + os.rename(tailed_filename, new_filename) + time.sleep(WAIT_TIME) + + assert appended_lines == [ + "Added line 1", + "Added line 2", + ] + + with open(new_filename, 'a+') as f: + f.write(" - end of line 2\n") + time.sleep(WAIT_TIME) + + assert appended_lines == [ + "Added line 1", + "Added line 2", + ] + + assert not os.path.exists(tailed_filename) + + with open(tailed_filename, 'w+') as f: + f.write("Recreated file - text line 1\n") + f.write("Recreated file - text line 2\n") + time.sleep(WAIT_TIME) + + assert os.path.exists(tailed_filename) + + assert appended_lines == [ + "Added line 1", + "Added line 2", + "Recreated file - text line 1", + "Recreated file - text line 2", + ] + + sft.close() + + observer.stop() + + os.unlink(new_filename) + os.unlink(tailed_filename) + + +def test_non_watched_file_moved(): + tailed_filename = 'tailed_file_to_move.txt' + not_tailed_filename = f'not_{tailed_filename}' + new_not_tailed_filename = not_tailed_filename.replace('_to_move.txt', '_moved.txt') + + if os.path.exists(not_tailed_filename): + os.unlink(not_tailed_filename) + if os.path.exists(new_not_tailed_filename): + os.unlink(new_not_tailed_filename) + if os.path.exists(tailed_filename): + os.unlink(tailed_filename) + + with open(not_tailed_filename, 'w+') as f: + f.write("Text here will not be monitored\n") + + with open(tailed_filename, 'w+') as f: + f.write("Preexisting text line 1\n") + f.write("Preexisting text line 2\n") + + appended_lines = [] + append_to_list_partial = functools.partial(append_to_list, appended_lines) + + observer = Observer() + + sft = SingleFileTail(tailed_filename, append_to_list_partial, + observer=observer, logger=logger) + + observer.start() + + with open(tailed_filename, 'a+') as f: + f.write("Added line 1\n") + time.sleep(WAIT_TIME) + + assert appended_lines == [ + "Added line 1", + ] + + os.replace(not_tailed_filename, new_not_tailed_filename) + time.sleep(WAIT_TIME) + + assert appended_lines == [ + "Added line 1", + ] + + sft.close() + + observer.stop() + + os.unlink(new_not_tailed_filename) + os.unlink(tailed_filename) + + +def test_watched_file_deleted(): + tailed_filename = 'tailed_file_deleted.txt' + with open(tailed_filename, 'w+') as f: + f.write("Preexisting text line 1\n") + f.write("Preexisting text line 2\n") + + appended_lines = [] + append_to_list_partial = functools.partial(append_to_list, appended_lines) + + observer = Observer() + + sft = SingleFileTail(tailed_filename, append_to_list_partial, + observer=observer, logger=logger) + + observer.start() + time.sleep(WAIT_TIME) + + with open(tailed_filename, 'a+') as f: + f.write("Added line 1\n") + time.sleep(WAIT_TIME) + + assert appended_lines == [ + "Added line 1", + ] + + os.unlink(tailed_filename) + time.sleep(WAIT_TIME) + + assert appended_lines == [ + "Added line 1", + ] + + sft.close() + + observer.stop() + + +def test_watched_file_immediately_deleted(): + tailed_filename = 'tailed_file_deleted.txt' + with open(tailed_filename, 'w+') as f: + f.write("Preexisting text line 1\n") + f.write("Preexisting text line 2\n") + + appended_lines = [] + append_to_list_partial = functools.partial(append_to_list, appended_lines) + + observer = Observer() + + sft = SingleFileTail(tailed_filename, append_to_list_partial, + observer=observer, logger=logger) + + observer.start() + time.sleep(WAIT_TIME) + + os.unlink(tailed_filename) + + assert appended_lines == [] + + sft.close() + + observer.stop() + + +if __name__ == '__main__': + import sys + + logger.setLevel(logging.DEBUG) + + handler = logging.StreamHandler(sys.stderr) + handler.setLevel(logging.DEBUG) + formatter = logging.Formatter('%(name)s: %(levelname)s: %(message)s') + + logger.addHandler(handler) + + test_read_chunk_from_nonexistent_file() diff --git a/contrib/linux/tests/test_tail_manager.py b/contrib/linux/tests/test_tail_manager.py new file mode 100644 index 0000000000..9e181135ac --- /dev/null +++ b/contrib/linux/tests/test_tail_manager.py @@ -0,0 +1,647 @@ +#!/usr/bin/env python + +# Copyright 2020 The StackStorm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import functools +import logging +import os +import pathlib +import time + +from watchdog.observers import Observer + +from file_watch_sensor import TailManager + +WAIT_TIME = 1 + +logger = logging.getLogger(__name__) + + +# Helper function + +def append_to_list(list_to_append, path, element): + logger.debug(f"Appending ({path}):\n{element} to {list_to_append}") + list_to_append.append(element) + + +def test_initialized_without_logger(): + try: + TailManager() + except Exception as e: + expected_message = "TailManager was initialized without a logger" + if hasattr(e, 'message') and e.message != expected_message: + raise e + else: + raise AssertionError("TailManager initialized fine without a " + "logger parameter") + + +def test_append_to_watched_file_with_absolute_path(): + tailed_filename = (pathlib.Path.cwd() / pathlib.Path('tailed_file.txt')).resolve() + with open(tailed_filename, 'w+') as f: + f.write("Preexisting text line 1\n") + f.write("Preexisting text line 2\n") + + appended_lines = [] + append_to_list_partial = functools.partial(append_to_list, appended_lines) + + tm = TailManager(logger=logger) + tm.tail_file(tailed_filename, handler=append_to_list_partial) + tm.start() + + with open(tailed_filename, 'a+') as f: + f.write("Added line 1\n") + time.sleep(WAIT_TIME) + + assert appended_lines == [ + "Added line 1", + ] + + tm.stop_tailing_file(tailed_filename, handler=append_to_list_partial) + + tm.stop() + + os.unlink(tailed_filename) + + +def test_not_watched_file(): + tailed_filename = 'tailed_file.txt' + not_tailed_filename = 'not_tailed_file.txt' + new_not_tailed_filename = not_tailed_filename.replace('.txt', '_moved.txt') + with open(tailed_filename, 'w+') as f: + f.write("Preexisting text line 1\n") + f.write("Preexisting text line 2\n") + + appended_lines = [] + append_to_list_partial = functools.partial(append_to_list, appended_lines) + + tm = TailManager(logger=logger) + tm.tail_file(tailed_filename, handler=append_to_list_partial) + tm.start() + + with open(tailed_filename, 'a+') as f: + f.write("Added line 1\n") + time.sleep(WAIT_TIME) + + assert appended_lines == [ + "Added line 1", + ] + + with open(not_tailed_filename, 'a+') as f: + f.write("Added line 1 - not tailed\n") + time.sleep(WAIT_TIME) + + assert appended_lines == [ + "Added line 1", + ] + + os.replace(not_tailed_filename, new_not_tailed_filename) + time.sleep(WAIT_TIME) + + assert appended_lines == [ + "Added line 1", + ] + + tm.stop_tailing_file(tailed_filename, handler=append_to_list_partial) + + tm.stop() + time.sleep(WAIT_TIME) + + assert appended_lines == [ + "Added line 1", + ] + + os.unlink(tailed_filename) + os.unlink(new_not_tailed_filename) + + +def test_watch_nonexistent_file(): + tailed_filename = 'tailed_file.txt' + + if os.path.exists(tailed_filename): + os.unlink(tailed_filename) + + appended_lines = [] + append_to_list_partial = functools.partial(append_to_list, appended_lines) + + tm = TailManager(logger=logger) + tm.tail_file(tailed_filename, handler=append_to_list_partial) + tm.start() + time.sleep(WAIT_TIME) + + assert appended_lines == [] + + with open(tailed_filename, 'w+') as f: + f.write("Added line 1\n") + time.sleep(WAIT_TIME) + + assert appended_lines == [ + "Added line 1", + ] + + tm.stop_tailing_file(tailed_filename, handler=append_to_list_partial) + + tm.stop() + time.sleep(WAIT_TIME) + + os.unlink(tailed_filename) + + +def test_follow_watched_file_moved(): + tailed_filename = 'tailed_file_to_move.txt' + new_filename = tailed_filename.replace('_to_move.txt', '_moved.txt') + + if os.path.exists(new_filename): + os.unlink(new_filename) + if os.path.exists(tailed_filename): + os.unlink(tailed_filename) + + with open(tailed_filename, 'w+') as f: + f.write("Preexisting text line 1\n") + f.write("Preexisting text line 2\n") + + appended_lines = [] + append_to_list_partial = functools.partial(append_to_list, appended_lines) + + tm = TailManager(logger=logger) + tm.tail_file(tailed_filename, handler=append_to_list_partial, follow=True) + tm.start() + + with open(tailed_filename, 'a+') as f: + f.write("Added line 1\n") + time.sleep(WAIT_TIME) + + assert appended_lines == [ + "Added line 1", + ] + + with open(tailed_filename, 'a+') as f: + f.write("Added line 2") # No newline + time.sleep(WAIT_TIME) + + assert appended_lines == [ + "Added line 1", + ] + + os.replace(tailed_filename, new_filename) + time.sleep(WAIT_TIME) + + assert appended_lines == [ + "Added line 1", + ] + + with open(new_filename, 'a+') as f: + f.write(" - end of line 2\n") + time.sleep(WAIT_TIME) + + assert appended_lines == [ + "Added line 1", + "Added line 2 - end of line 2", + ] + + with open(tailed_filename, 'w+') as f: + f.write("New file - text line 1\n") + f.write("New file - text line 2\n") + time.sleep(WAIT_TIME) + + assert appended_lines == [ + "Added line 1", + "Added line 2 - end of line 2", + ] + + tm.stop_tailing_file(tailed_filename, handler=append_to_list_partial) + + tm.stop() + + os.unlink(new_filename) + os.unlink(tailed_filename) + + +def test_not_followed_watched_file_moved(): + tailed_filename = 'tailed_file_to_move.txt' + new_filename = tailed_filename.replace('_to_move.txt', '_moved.txt') + + if os.path.exists(new_filename): + os.unlink(new_filename) + if os.path.exists(tailed_filename): + os.unlink(tailed_filename) + + with open(tailed_filename, 'w+') as f: + f.write("Preexisting text line 1\n") + f.write("Preexisting text line 2\n") + + appended_lines = [] + append_to_list_partial = functools.partial(append_to_list, appended_lines) + + tm = TailManager(logger=logger) + tm.tail_file(tailed_filename, handler=append_to_list_partial, follow=False) + tm.start() + + with open(tailed_filename, 'a+') as f: + f.write("Added line 1\n") + time.sleep(WAIT_TIME) + + assert appended_lines == [ + "Added line 1", + ] + + with open(tailed_filename, 'a+') as f: + f.write("Added line 2") # No newline + time.sleep(WAIT_TIME) + + assert appended_lines == [ + "Added line 1", + ] + + os.replace(tailed_filename, new_filename) + time.sleep(WAIT_TIME) + + assert appended_lines == [ + "Added line 1", + "Added line 2", + ] + + with open(new_filename, 'a+') as f: + f.write(" - end of line 2\n") + time.sleep(WAIT_TIME) + + assert appended_lines == [ + "Added line 1", + "Added line 2", + ] + + with open(tailed_filename, 'w+') as f: + f.write("Recreated file - text line 1\n") + f.write("Recreated file - text line 2\n") + time.sleep(WAIT_TIME) + + assert appended_lines == [ + "Added line 1", + "Added line 2", + "Recreated file - text line 1", + "Recreated file - text line 2", + ] + + tm.stop_tailing_file(tailed_filename, handler=append_to_list_partial) + + tm.stop() + + os.unlink(new_filename) + os.unlink(tailed_filename) + + +def test_non_watched_file_moved(): + tailed_filename = 'tailed_file_to_move.txt' + not_tailed_filename = f'not_{tailed_filename}' + new_not_tailed_filename = not_tailed_filename.replace('_to_move.txt', '_moved.txt') + + if os.path.exists(not_tailed_filename): + os.unlink(not_tailed_filename) + if os.path.exists(new_not_tailed_filename): + os.unlink(new_not_tailed_filename) + if os.path.exists(tailed_filename): + os.unlink(tailed_filename) + + with open(not_tailed_filename, 'w+') as f: + f.write("Text here will not be monitored\n") + + with open(tailed_filename, 'w+') as f: + f.write("Preexisting text line 1\n") + f.write("Preexisting text line 2\n") + + appended_lines = [] + append_to_list_partial = functools.partial(append_to_list, appended_lines) + + tm = TailManager(logger=logger) + tm.tail_file(tailed_filename, handler=append_to_list_partial) + tm.start() + + with open(tailed_filename, 'a+') as f: + f.write("Added line 1\n") + time.sleep(WAIT_TIME) + + assert appended_lines == [ + "Added line 1", + ] + + os.replace(not_tailed_filename, new_not_tailed_filename) + time.sleep(WAIT_TIME) + + assert appended_lines == [ + "Added line 1", + ] + + tm.stop_tailing_file(tailed_filename, handler=append_to_list_partial) + + tm.stop() + + os.unlink(new_not_tailed_filename) + os.unlink(tailed_filename) + + +def test_watched_file_deleted(): + tailed_filename = 'tailed_file_deleted.txt' + with open(tailed_filename, 'w+') as f: + f.write("Preexisting text line 1\n") + f.write("Preexisting text line 2\n") + + appended_lines = [] + append_to_list_partial = functools.partial(append_to_list, appended_lines) + + tm = TailManager(logger=logger) + tm.tail_file(tailed_filename, handler=append_to_list_partial) + tm.start() + + with open(tailed_filename, 'a+') as f: + f.write("Added line 1\n") + time.sleep(WAIT_TIME) + + assert appended_lines == [ + "Added line 1", + ] + + os.unlink(tailed_filename) + + assert appended_lines == [ + "Added line 1", + ] + + tm.stop() + + +def test_watched_file_immediately_deleted(): + tailed_filename = 'tailed_file_deleted.txt' + with open(tailed_filename, 'w+') as f: + f.write("Preexisting text line 1\n") + f.write("Preexisting text line 2\n") + + appended_lines = [] + append_to_list_partial = functools.partial(append_to_list, appended_lines) + + tm = TailManager(logger=logger) + tm.tail_file(tailed_filename, handler=append_to_list_partial) + tm.start() + + os.unlink(tailed_filename) + + tm.stop() + + +def test_append_to_watched_file(): + tailed_filename = 'tailed_file.txt' + with open(tailed_filename, 'w+') as f: + f.write("Preexisting text line 1\n") + f.write("Preexisting text line 2\n") + + appended_lines = [] + append_to_list_partial = functools.partial(append_to_list, appended_lines) + + tm = TailManager(logger=logger) + tm.tail_file(tailed_filename, handler=append_to_list_partial) + tm.start() + + with open(tailed_filename, 'a+') as f: + f.write("Added line 1\n") + time.sleep(WAIT_TIME) + + assert appended_lines == [ + "Added line 1", + ] + + with open(tailed_filename, 'a+') as f: + f.write("Added line 2\n") + time.sleep(WAIT_TIME) + + assert appended_lines == [ + "Added line 1", + "Added line 2", + ] + + with open(tailed_filename, 'a+') as f: + f.write("Start of added partial line 1") + time.sleep(WAIT_TIME) + + assert appended_lines == [ + "Added line 1", + "Added line 2", + ] + + with open(tailed_filename, 'a+') as f: + f.write(" - finished partial line 1\nStart of added partial line 2") + time.sleep(WAIT_TIME) + + assert appended_lines == [ + "Added line 1", + "Added line 2", + "Start of added partial line 1 - finished partial line 1", + ] + + with open(tailed_filename, 'a+') as f: + f.write(" - finished partial line 2\n") + time.sleep(WAIT_TIME) + + assert appended_lines == [ + "Added line 1", + "Added line 2", + "Start of added partial line 1 - finished partial line 1", + "Start of added partial line 2 - finished partial line 2", + ] + + with open(tailed_filename, 'a+') as f: + f.write("Final line without a newline") + time.sleep(WAIT_TIME) + + tm.stop_tailing_file(tailed_filename, handler=append_to_list_partial) + + tm.stop() + + time.sleep(WAIT_TIME) + assert appended_lines == [ + "Added line 1", + "Added line 2", + "Start of added partial line 1 - finished partial line 1", + "Start of added partial line 2 - finished partial line 2", + "Final line without a newline", + ] + + os.unlink(tailed_filename) + + +def test_tail_file_twice(): + tailed_filename = 'tailed_file.txt' + with open(tailed_filename, 'w+') as f: + f.write("Preexisting text line 1\n") + f.write("Preexisting text line 2\n") + + appended_lines = [] + append_to_list_partial = functools.partial(append_to_list, appended_lines) + + tm = TailManager(logger=logger) + tm.tail_file(tailed_filename, handler=append_to_list_partial) + tm.tail_file(tailed_filename, handler=append_to_list_partial) + tm.start() + + with open(tailed_filename, 'a+') as f: + f.write("Added line 1\n") + time.sleep(WAIT_TIME) + + assert appended_lines == [ + "Added line 1", + ] + + with open(tailed_filename, 'a+') as f: + f.write("Added line 2\n") + time.sleep(WAIT_TIME) + + assert appended_lines == [ + "Added line 1", + "Added line 2", + ] + + with open(tailed_filename, 'a+') as f: + f.write("Start of added partial line 1") + time.sleep(WAIT_TIME) + + assert appended_lines == [ + "Added line 1", + "Added line 2", + ] + + with open(tailed_filename, 'a+') as f: + f.write(" - finished partial line 1\nStart of added partial line 2") + time.sleep(WAIT_TIME) + + assert appended_lines == [ + "Added line 1", + "Added line 2", + "Start of added partial line 1 - finished partial line 1", + ] + + with open(tailed_filename, 'a+') as f: + f.write(" - finished partial line 2\n") + time.sleep(WAIT_TIME) + + assert appended_lines == [ + "Added line 1", + "Added line 2", + "Start of added partial line 1 - finished partial line 1", + "Start of added partial line 2 - finished partial line 2", + ] + + with open(tailed_filename, 'a+') as f: + f.write("Final line without a newline") + time.sleep(WAIT_TIME) + + tm.stop_tailing_file(tailed_filename, handler=append_to_list_partial) + + tm.stop() + + time.sleep(WAIT_TIME) + assert appended_lines == [ + "Added line 1", + "Added line 2", + "Start of added partial line 1 - finished partial line 1", + "Start of added partial line 2 - finished partial line 2", + "Final line without a newline", + ] + + os.unlink(tailed_filename) + + +def test_stop(): + tailed_filename = 'tailed_file_stop.txt' + with open(tailed_filename, 'w+') as f: + f.write("Preexisting text line 1\n") + f.write("Preexisting text line 2\n") + + appended_lines = [] + append_to_list_partial = functools.partial(append_to_list, appended_lines) + + tm = TailManager(logger=logger) + tm.tail_file(tailed_filename, handler=append_to_list_partial) + tm.start() + + with open(tailed_filename, 'a+') as f: + f.write("Added line 1\n") + time.sleep(WAIT_TIME) + + assert appended_lines == [ + "Added line 1", + ] + + with open(tailed_filename, 'a+') as f: + f.write("Final line without a newline") + time.sleep(WAIT_TIME) + + tm.stop() + + time.sleep(WAIT_TIME) + assert appended_lines == [ + "Added line 1", + "Final line without a newline", + ] + + os.unlink(tailed_filename) + + +def test_stop_twice(): + tailed_filename = 'tailed_file_stop.txt' + with open(tailed_filename, 'w+') as f: + f.write("Preexisting text line 1\n") + f.write("Preexisting text line 2\n") + + appended_lines = [] + append_to_list_partial = functools.partial(append_to_list, appended_lines) + + tm = TailManager(logger=logger) + tm.tail_file(tailed_filename, handler=append_to_list_partial) + tm.start() + + with open(tailed_filename, 'a+') as f: + f.write("Added line 1\n") + time.sleep(WAIT_TIME) + + assert appended_lines == [ + "Added line 1", + ] + + with open(tailed_filename, 'a+') as f: + f.write("Final line without a newline") + time.sleep(WAIT_TIME) + + tm.stop() + tm.stop() + + time.sleep(WAIT_TIME) + assert appended_lines == [ + "Added line 1", + "Final line without a newline", + ] + + os.unlink(tailed_filename) + + +if __name__ == '__main__': + import sys + + logger.setLevel(logging.DEBUG) + + handler = logging.StreamHandler(sys.stderr) + handler.setLevel(logging.DEBUG) + formatter = logging.Formatter('%(name)s: %(levelname)s: %(message)s') + + logger.addHandler(handler) + + test_single_file_tail_append_to_watched_file_with_relative_path() From 42799860a444968247174b8ee10e8a1409e5e704 Mon Sep 17 00:00:00 2001 From: blag Date: Wed, 7 Apr 2021 18:03:26 -0700 Subject: [PATCH 19/35] Add .vagrant/ to .gitignore --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 9e174a3e48..4c329e3023 100644 --- a/.gitignore +++ b/.gitignore @@ -35,6 +35,9 @@ conf/st2.travis.conf # generated GitHub Actions conf conf/st2.githubactions.conf +# Vagrant +.vagrant/ + # Installer logs pip-log.txt From 56017439acf0bd0b1bc10bc5da32aaea9121102c Mon Sep 17 00:00:00 2001 From: blag Date: Mon, 12 Apr 2021 12:49:46 -0700 Subject: [PATCH 20/35] Ignore a few more virtualenv directories --- pyproject.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 1889c6a5da..e5eddb4eab 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,6 +7,9 @@ exclude = ''' /( | \.git | \.virtualenv + | venv + | virtualenv + | virtualenv-osx | __pycache__ | test_content_version )/ From 0c10559073c0a255cc1e7b367830de2ea2a89c1f Mon Sep 17 00:00:00 2001 From: blag Date: Mon, 12 Apr 2021 12:49:59 -0700 Subject: [PATCH 21/35] Make pylint happy --- contrib/linux/tests/test_file_watch_sensor.py | 4 +--- contrib/linux/tests/test_tail_manager.py | 3 ++- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/contrib/linux/tests/test_file_watch_sensor.py b/contrib/linux/tests/test_file_watch_sensor.py index c42a110306..dfb94baadb 100644 --- a/contrib/linux/tests/test_file_watch_sensor.py +++ b/contrib/linux/tests/test_file_watch_sensor.py @@ -129,7 +129,7 @@ def test_file_watch_sensor_without_trigger_filepath(): time.sleep(WAIT_TIME) - result = fws.add_trigger({ + fws.add_trigger({ 'id': 'asdf.adsfasdf-asdf-asdf-asdfasdfasdf', 'pack': 'linux', 'name': 'asdf.adsfasdf-asdf-asdf-asdfasdfasdf', @@ -142,8 +142,6 @@ def test_file_watch_sensor_without_trigger_filepath(): }, }) - assert result is None - def test_file_watch_sensor_without_trigger_ref(): mock_sensor_service = mock.MagicMock() diff --git a/contrib/linux/tests/test_tail_manager.py b/contrib/linux/tests/test_tail_manager.py index 9e181135ac..a024961dd0 100644 --- a/contrib/linux/tests/test_tail_manager.py +++ b/contrib/linux/tests/test_tail_manager.py @@ -41,7 +41,8 @@ def test_initialized_without_logger(): TailManager() except Exception as e: expected_message = "TailManager was initialized without a logger" - if hasattr(e, 'message') and e.message != expected_message: + exc_msg = getattr(e, 'message', e.args[0]) + if exc_msg != expected_message: raise e else: raise AssertionError("TailManager initialized fine without a " From f8a305659d8e421882024e19b793d35eca1675c4 Mon Sep 17 00:00:00 2001 From: blag Date: Mon, 12 Apr 2021 13:11:44 -0700 Subject: [PATCH 22/35] Run all tests when run directly --- contrib/linux/tests/test_file_watch_sensor.py | 2 -- contrib/linux/tests/test_single_file_tail.py | 13 +++++++++++++ contrib/linux/tests/test_tail_manager.py | 14 +++++++++++++- 3 files changed, 26 insertions(+), 3 deletions(-) diff --git a/contrib/linux/tests/test_file_watch_sensor.py b/contrib/linux/tests/test_file_watch_sensor.py index dfb94baadb..be7484318f 100644 --- a/contrib/linux/tests/test_file_watch_sensor.py +++ b/contrib/linux/tests/test_file_watch_sensor.py @@ -199,8 +199,6 @@ def test_file_watch_sensor_without_trigger_ref(): # logger.addHandler(handler) - # test_single_file_tail_watched_file_deleted() - test_file_watch_sensor() test_file_watch_sensor_without_trigger_filepath() test_file_watch_sensor_without_trigger_ref() diff --git a/contrib/linux/tests/test_single_file_tail.py b/contrib/linux/tests/test_single_file_tail.py index 18fb5f23a6..e1cb4879e0 100644 --- a/contrib/linux/tests/test_single_file_tail.py +++ b/contrib/linux/tests/test_single_file_tail.py @@ -600,4 +600,17 @@ def test_watched_file_immediately_deleted(): logger.addHandler(handler) + test_read_chunk_over_multibyte_character_boundary() + test_read_chunk_with_bad_utf8_character() test_read_chunk_from_nonexistent_file() + test_initialize_without_logger() + test_append_to_watched_file_with_absolute_path() + test_append_to_watched_file_with_relative_path() + test_append_to_watched_file_observer_start_first() + test_not_watched_file() + test_watch_nonexistent_file() + test_follow_watched_file_moved() + test_not_followed_watched_file_moved() + test_non_watched_file_moved() + test_watched_file_deleted() + test_watched_file_immediately_deleted() diff --git a/contrib/linux/tests/test_tail_manager.py b/contrib/linux/tests/test_tail_manager.py index a024961dd0..6664c59b92 100644 --- a/contrib/linux/tests/test_tail_manager.py +++ b/contrib/linux/tests/test_tail_manager.py @@ -645,4 +645,16 @@ def test_stop_twice(): logger.addHandler(handler) - test_single_file_tail_append_to_watched_file_with_relative_path() + test_initialized_without_logger() + test_append_to_watched_file_with_absolute_path() + test_not_watched_file() + test_watch_nonexistent_file() + test_follow_watched_file_moved() + test_not_followed_watched_file_moved() + test_non_watched_file_moved() + test_watched_file_deleted() + test_watched_file_immediately_deleted() + test_append_to_watched_file() + test_tail_file_twice() + test_stop() + test_stop_twice() From 5f3cf9acad5742998088335c867d4a61df8f8030 Mon Sep 17 00:00:00 2001 From: blag Date: Mon, 12 Apr 2021 13:16:12 -0700 Subject: [PATCH 23/35] Make black happy --- contrib/linux/sensors/file_watch_sensor.py | 115 +++++++---- contrib/linux/tests/test_file_watch_sensor.py | 131 +++++++------ contrib/linux/tests/test_single_file_tail.py | 184 ++++++++++-------- contrib/linux/tests/test_tail_manager.py | 132 ++++++------- 4 files changed, 314 insertions(+), 248 deletions(-) diff --git a/contrib/linux/sensors/file_watch_sensor.py b/contrib/linux/sensors/file_watch_sensor.py index 9372e0f095..bdf7594208 100644 --- a/contrib/linux/sensors/file_watch_sensor.py +++ b/contrib/linux/sensors/file_watch_sensor.py @@ -26,6 +26,7 @@ try: from st2reactor.sensor.base import Sensor except ImportError: + class Sensor: def __init__(self, *args, sensor_service=None, config=None, **kwargs): self.sensor_service = sensor_service @@ -39,6 +40,7 @@ class EventHandler(FileSystemEventHandler): individual files, since the directory events will include events for individual files. """ + def __init__(self, *args, callbacks=None, **kwargs): self.callbacks = callbacks or {} @@ -47,22 +49,22 @@ def dispatch(self, event): super().dispatch(event) def on_created(self, event): - cb = self.callbacks.get('created') + cb = self.callbacks.get("created") if cb: cb(event=event) def on_modified(self, event): - cb = self.callbacks.get('modified') + cb = self.callbacks.get("modified") if cb: cb(event=event) def on_moved(self, event): - cb = self.callbacks.get('moved') + cb = self.callbacks.get("moved") if cb: cb(event=event) def on_deleted(self, event): - cb = self.callbacks.get('deleted') + cb = self.callbacks.get("deleted") if cb: cb(event=event) @@ -98,8 +100,17 @@ class SingleFileTail(object): where one file is quickly created and/or updated may trigger race conditions and therefore unpredictable behavior. """ - def __init__(self, path, handler, follow=False, read_all=False, - observer=None, logger=None, fd=None): + + def __init__( + self, + path, + handler, + follow=False, + read_all=False, + observer=None, + logger=None, + fd=None, + ): if logger is None: raise Exception("SingleFileTail was initialized without a logger") @@ -108,7 +119,7 @@ def __init__(self, path, handler, follow=False, read_all=False, self.handler = handler self.follow = follow self.read_all = read_all - self.buffer = '' + self.buffer = "" self.observer = observer or Observer() self.logger = logger self.watch = None @@ -146,7 +157,7 @@ def read_chunk(self, fd, chunk_size=1024): # If the 1024 bytes cuts the line off in the middle of a multi-byte # utf-8 character then decoding will raise an UnicodeDecodeError. try: - buffer = buffer.decode(encoding='utf8') + buffer = buffer.decode(encoding="utf8") except UnicodeDecodeError as e: # Grab the first few bytes of the partial character # e.start is the first byte of the decoding issue @@ -174,7 +185,7 @@ def read_chunk(self, fd, chunk_size=1024): buff = os.read(fd, number_of_bytes_to_read) if len(buff) == number_of_bytes_to_read: buffer += buff - return buffer.decode(encoding='utf8') + return buffer.decode(encoding="utf8") # If we did not successfully read a complete character, there's # nothing else we can really do but reraise the exception @@ -195,11 +206,13 @@ def read(self, event=None): if self.buffer: self.logger.debug(f"Appending to existing buffer: '{self.buffer}'") buff = self.buffer + buff - self.buffer = '' + self.buffer = "" lines = buff.splitlines(True) # If the last character of the last line is not a newline - if lines and lines[-1] and lines[-1][-1] != '\n': # Incomplete line in the buffer + if ( + lines and lines[-1] and lines[-1][-1] != "\n" + ): # Incomplete line in the buffer self.logger.debug(f"Saving partial line in the buffer: '{lines[-1]}'") self.buffer = lines[-1] # Save the last line fragment lines = lines[:-1] @@ -212,7 +225,9 @@ def reopen_and_read(self, event=None, skip_to_end=False): # Directory watches will fire events for unrelated files # Ignore all events except those for our path if event and self.get_event_src_path(event) != self.abs_path: - self.logger.debug(f"Ignoring event for non-tracked file: '{event.src_path}'") + self.logger.debug( + f"Ignoring event for non-tracked file: '{event.src_path}'" + ) return # Guard against this being called twice - happens sometimes with inotify @@ -267,20 +282,22 @@ def open(self, event=None, seek_to=None): else: self.fd = os.open(self.path, os.O_RDONLY | os.O_NONBLOCK) - if self.read_all or seek_to == 'start': + if self.read_all or seek_to == "start": self.logger.debug("Seeking to start") os.lseek(self.fd, 0, os.SEEK_SET) - if not self.read_all or seek_to == 'end': + if not self.read_all or seek_to == "end": self.logger.debug("Seeking to end") os.lseek(self.fd, 0, os.SEEK_END) - file_event_handler = EventHandler(callbacks={ - 'created': self.open, - 'deleted': self.close, - 'modified': self.read, - 'moved': self.reopen_and_read, - }) + file_event_handler = EventHandler( + callbacks={ + "created": self.open, + "deleted": self.close, + "modified": self.read, + "moved": self.reopen_and_read, + } + ) self.logger.debug(f"Scheduling watch on file: '{self.path}'") self.watch = self.observer.schedule(file_event_handler, self.path) @@ -289,14 +306,22 @@ def open(self, event=None, seek_to=None): # Avoid watching this twice self.logger.debug(f"Parent watch: {self.parent_watch}") if not self.parent_watch: - dir_event_handler = EventHandler(callbacks={ - 'created': self.open_and_read, - 'moved': self.reopen_and_read, - }) - - self.logger.debug(f"Scheduling watch on parent directory: '{self.parent_dir}'") - self.parent_watch = self.observer.schedule(dir_event_handler, self.parent_dir) - self.logger.debug(f"Scheduled watch on parent directory: '{self.parent_dir}'") + dir_event_handler = EventHandler( + callbacks={ + "created": self.open_and_read, + "moved": self.reopen_and_read, + } + ) + + self.logger.debug( + f"Scheduling watch on parent directory: '{self.parent_dir}'" + ) + self.parent_watch = self.observer.schedule( + dir_event_handler, self.parent_dir + ) + self.logger.debug( + f"Scheduled watch on parent directory: '{self.parent_dir}'" + ) def close(self, event=None, emit_remaining=True, end_parent_watch=True): self.logger.debug(f"Closing single file tail on '{self.path}'") @@ -304,7 +329,7 @@ def close(self, event=None, emit_remaining=True, end_parent_watch=True): if self.buffer and emit_remaining: self.logger.debug(f"Emitting remaining partial line: '{self.buffer}'") self.handler(self.path, self.buffer) - self.buffer = '' + self.buffer = "" if self.parent_watch and end_parent_watch: self.logger.debug(f"Unscheduling parent directory watch: {self.parent_dir}") self.observer.unschedule(self.parent_watch) @@ -338,10 +363,14 @@ def __init__(self, *args, logger=None, **kwargs): def tail_file(self, path, handler, follow=False, read_all=False): if handler not in self.tails.setdefault(path, {}): self.logger.debug(f"Tailing single file: {path}") - sft = SingleFileTail(path, handler, - follow=follow, read_all=read_all, - observer=self.observer, - logger=self.logger) + sft = SingleFileTail( + path, + handler, + follow=follow, + read_all=read_all, + observer=self.observer, + logger=self.logger, + ) self.tails[path][handler] = sft def stop_tailing_file(self, path, handler): @@ -398,10 +427,10 @@ def add_trigger(self, trigger): self.logger.error('Received trigger type without "file_path" field.') return - self.trigger = trigger.get('ref', None) + self.trigger = trigger.get("ref", None) if not self.trigger: - raise Exception('Trigger %s did not contain a ref.' % trigger) + raise Exception("Trigger %s did not contain a ref." % trigger) self.tail_manager.tail_file(file_path, self._handle_line) self.logger.info('Added file "%s"' % (file_path)) @@ -425,16 +454,19 @@ def remove_trigger(self, trigger): def _handle_line(self, file_path, line): payload = { - 'file_path': file_path, - 'file_name': pathlib.Path(file_path).name, - 'line': line + "file_path": file_path, + "file_name": pathlib.Path(file_path).name, + "line": line, } - self.logger.debug('Sending payload %s for trigger %s to sensor_service.', - payload, self.trigger) + self.logger.debug( + "Sending payload %s for trigger %s to sensor_service.", + payload, + self.trigger, + ) self.sensor_service.dispatch(trigger=self.trigger, payload=payload) -if __name__ == '__main__': +if __name__ == "__main__": logger = logging.getLogger(__name__) tm = TailManager(logger=logger) tm.tail_file(__file__, handler=print) @@ -443,4 +475,5 @@ def _handle_line(self, file_path, line): def halt(sig, frame): tm.stop() sys.exit(0) + signal.signal(signal.SIGINT, halt) diff --git a/contrib/linux/tests/test_file_watch_sensor.py b/contrib/linux/tests/test_file_watch_sensor.py index be7484318f..08edf71465 100644 --- a/contrib/linux/tests/test_file_watch_sensor.py +++ b/contrib/linux/tests/test_file_watch_sensor.py @@ -35,12 +35,13 @@ def test_file_watch_sensor(): mock_sensor_service = mock.MagicMock() mock_logger = mock.MagicMock() - filename = 'test.txt' + filename = "test.txt" filepath = pathlib.Path(filename).absolute().resolve() filepath.touch() - fws = FileWatchSensor(sensor_service=mock_sensor_service, config={}, - logger=mock_logger) + fws = FileWatchSensor( + sensor_service=mock_sensor_service, config={}, logger=mock_logger + ) time.sleep(WAIT_TIME) @@ -54,48 +55,50 @@ def test_file_watch_sensor(): time.sleep(WAIT_TIME) - fws.add_trigger({ - 'id': 'asdf.adsfasdf-asdf-asdf-asdfasdfasdf', - 'pack': 'linux', - 'name': 'asdf.adsfasdf-asdf-asdf-asdfasdfasdf', - 'ref': 'linux.asdf.adsfasdf-asdf-asdf-asdfasdfasdf', - 'uid': 'trigger:linux:asdf.adsfasdf-asdf-asdf-asdfasdfasdf', - 'type': 'linux.file_watch.line', - 'parameters': { - 'file_path': filepath, - 'follow': True, - }, - }) + fws.add_trigger( + { + "id": "asdf.adsfasdf-asdf-asdf-asdfasdfasdf", + "pack": "linux", + "name": "asdf.adsfasdf-asdf-asdf-asdfasdfasdf", + "ref": "linux.asdf.adsfasdf-asdf-asdf-asdfasdfasdf", + "uid": "trigger:linux:asdf.adsfasdf-asdf-asdf-asdfasdfasdf", + "type": "linux.file_watch.line", + "parameters": { + "file_path": filepath, + "follow": True, + }, + } + ) time.sleep(WAIT_TIME) - with open(filepath, 'a') as f: + with open(filepath, "a") as f: f.write("Added line 1\n") time.sleep(WAIT_TIME) - with open(filepath, 'a') as f: + with open(filepath, "a") as f: f.write("Added line 2\n") time.sleep(WAIT_TIME) expected_calls = [ mock.call( - trigger='linux.asdf.adsfasdf-asdf-asdf-asdfasdfasdf', + trigger="linux.asdf.adsfasdf-asdf-asdf-asdfasdfasdf", payload={ - 'file_path': pathlib.PosixPath('/vagrant/contrib/linux/test.txt'), - 'file_name': 'test.txt', - 'line': 'Added line 1', + "file_path": pathlib.PosixPath("/vagrant/contrib/linux/test.txt"), + "file_name": "test.txt", + "line": "Added line 1", }, ), mock.call( - trigger='linux.asdf.adsfasdf-asdf-asdf-asdfasdfasdf', + trigger="linux.asdf.adsfasdf-asdf-asdf-asdfasdfasdf", payload={ - 'file_path': pathlib.PosixPath('/vagrant/contrib/linux/test.txt'), - 'file_name': 'test.txt', - 'line': 'Added line 2', + "file_path": pathlib.PosixPath("/vagrant/contrib/linux/test.txt"), + "file_name": "test.txt", + "line": "Added line 2", }, - ) + ), ] mock_sensor_service.dispatch.assert_has_calls(expected_calls, any_order=False) print(mock_logger.method_calls) @@ -110,12 +113,13 @@ def test_file_watch_sensor_without_trigger_filepath(): mock_sensor_service = mock.MagicMock() mock_logger = mock.MagicMock() - filename = 'test.txt' + filename = "test.txt" filepath = pathlib.Path(filename).absolute().resolve() filepath.touch() - fws = FileWatchSensor(sensor_service=mock_sensor_service, config={}, - logger=mock_logger) + fws = FileWatchSensor( + sensor_service=mock_sensor_service, config={}, logger=mock_logger + ) time.sleep(WAIT_TIME) @@ -129,30 +133,33 @@ def test_file_watch_sensor_without_trigger_filepath(): time.sleep(WAIT_TIME) - fws.add_trigger({ - 'id': 'asdf.adsfasdf-asdf-asdf-asdfasdfasdf', - 'pack': 'linux', - 'name': 'asdf.adsfasdf-asdf-asdf-asdfasdfasdf', - 'ref': 'linux.asdf.adsfasdf-asdf-asdf-asdfasdfasdf', - 'uid': 'trigger:linux:asdf.adsfasdf-asdf-asdf-asdfasdfasdf', - 'type': 'linux.file_watch.line', - 'parameters': { - # 'file_path': filepath, - 'follow': True, - }, - }) + fws.add_trigger( + { + "id": "asdf.adsfasdf-asdf-asdf-asdfasdfasdf", + "pack": "linux", + "name": "asdf.adsfasdf-asdf-asdf-asdfasdfasdf", + "ref": "linux.asdf.adsfasdf-asdf-asdf-asdfasdfasdf", + "uid": "trigger:linux:asdf.adsfasdf-asdf-asdf-asdfasdfasdf", + "type": "linux.file_watch.line", + "parameters": { + # 'file_path': filepath, + "follow": True, + }, + } + ) def test_file_watch_sensor_without_trigger_ref(): mock_sensor_service = mock.MagicMock() mock_logger = mock.MagicMock() - filename = 'test.txt' + filename = "test.txt" filepath = pathlib.Path(filename).absolute().resolve() filepath.touch() - fws = FileWatchSensor(sensor_service=mock_sensor_service, config={}, - logger=mock_logger) + fws = FileWatchSensor( + sensor_service=mock_sensor_service, config={}, logger=mock_logger + ) time.sleep(WAIT_TIME) @@ -167,30 +174,34 @@ def test_file_watch_sensor_without_trigger_ref(): time.sleep(WAIT_TIME) try: - fws.add_trigger({ - 'id': 'asdf.adsfasdf-asdf-asdf-asdfasdfasdf', - 'pack': 'linux', - 'name': 'asdf.adsfasdf-asdf-asdf-asdfasdfasdf', - # 'ref': 'linux.asdf.adsfasdf-asdf-asdf-asdfasdfasdf', - 'uid': 'trigger:linux:asdf.adsfasdf-asdf-asdf-asdfasdfasdf', - 'type': 'linux.file_watch.line', - 'parameters': { - 'file_path': filepath, - 'follow': True, - }, - }) + fws.add_trigger( + { + "id": "asdf.adsfasdf-asdf-asdf-asdfasdfasdf", + "pack": "linux", + "name": "asdf.adsfasdf-asdf-asdf-asdfasdfasdf", + # 'ref': 'linux.asdf.adsfasdf-asdf-asdf-asdfasdfasdf', + "uid": "trigger:linux:asdf.adsfasdf-asdf-asdf-asdfasdfasdf", + "type": "linux.file_watch.line", + "parameters": { + "file_path": filepath, + "follow": True, + }, + } + ) except Exception as e: # Make sure we ignore the right exception - if 'did not contain a ref' not in str(e): + if "did not contain a ref" not in str(e): raise e else: - raise AssertionError("FileWatchSensor.add_trigger() did not raise an " - "exception when passed a trigger without a ref") + raise AssertionError( + "FileWatchSensor.add_trigger() did not raise an " + "exception when passed a trigger without a ref" + ) finally: os.unlink(filepath) -if __name__ == '__main__': +if __name__ == "__main__": # logger.setLevel(logging.DEBUG) # handler = logging.StreamHandler(sys.stderr) diff --git a/contrib/linux/tests/test_single_file_tail.py b/contrib/linux/tests/test_single_file_tail.py index e1cb4879e0..1b3aeffe16 100644 --- a/contrib/linux/tests/test_single_file_tail.py +++ b/contrib/linux/tests/test_single_file_tail.py @@ -30,7 +30,7 @@ def test_read_chunk_over_multibyte_character_boundary(): - wide_characters = [None, None, '\u0130', '\u2050', '\U00088080'] + wide_characters = [None, None, "\u0130", "\u2050", "\U00088080"] for n in range(2, 5): yield from _gen_n_byte_character_tests(1024, n, wide_characters[n]) @@ -47,12 +47,12 @@ def _gen_n_byte_character_tests(chunk_size, n, char): def _run_n_byte_character_tests(chunk_size, n, length, char): - filename = f'chunk_boundary_{n}u_{length}.txt' + filename = f"chunk_boundary_{n}u_{length}.txt" - with open(filename, 'wb+') as f: + with open(filename, "wb+") as f: # Write out a file that is of the given length # aaaaaa...aaa\x82 - f.write(('a' * (length - n) + char).encode('utf-8')) + f.write(("a" * (length - n) + char).encode("utf-8")) fd = os.open(filename, os.O_RDONLY) @@ -64,24 +64,24 @@ def _run_n_byte_character_tests(chunk_size, n, length, char): os.unlink(filename) if length < chunk_size + n: - assert result == ('a' * (length - n) + char) + assert result == ("a" * (length - n) + char) else: - assert result == ('a' * chunk_size) + assert result == ("a" * chunk_size) def test_read_chunk_with_bad_utf8_character(): - filename = 'bad_utf8_character.txt' + filename = "bad_utf8_character.txt" - utf8_str = '\U00088080' - utf8_bytes = utf8_str.encode('utf-8') + utf8_str = "\U00088080" + utf8_bytes = utf8_str.encode("utf-8") chopped_utf8_bytes = utf8_bytes[:-1] - with open(filename, 'wb+') as f: + with open(filename, "wb+") as f: # Write out a file that is of the given length # aaaaaa...aaa\x82 - f.write(b'a') + f.write(b"a") f.write(chopped_utf8_bytes) - f.write(b'a') + f.write(b"a") fd = os.open(filename, os.O_RDONLY) @@ -101,9 +101,9 @@ def test_read_chunk_with_bad_utf8_character(): def test_read_chunk_from_nonexistent_file(): - filename = 'nonexistent_file.txt' + filename = "nonexistent_file.txt" - with open(filename, 'w+') as f: + with open(filename, "w+") as f: f.write("This file will not exist in a few moments") fd = os.open(filename, os.O_RDONLY) @@ -113,11 +113,12 @@ def test_read_chunk_from_nonexistent_file(): sft = SingleFileTail(None, None, fd=fd, logger=logger) - assert sft.read_chunk(fd=fd) == '' + assert sft.read_chunk(fd=fd) == "" # Helper function + def append_to_list(list_to_append, path, element): logger.debug(f"Appending ({path}):\n{element} to {list_to_append}") list_to_append.append(element) @@ -128,16 +129,17 @@ def test_initialize_without_logger(): SingleFileTail(None, None, fd=None) except Exception as e: expected_message = "SingleFileTail was initialized without a logger" - if hasattr(e, 'message') and e.message != expected_message: + if hasattr(e, "message") and e.message != expected_message: raise e else: - raise AssertionError("SingleFileTail initialized fine without a " - "logger parameter") + raise AssertionError( + "SingleFileTail initialized fine without a " "logger parameter" + ) def test_append_to_watched_file_with_absolute_path(): - tailed_filename = (pathlib.Path.cwd() / pathlib.Path('tailed_file.txt')).resolve() - with open(tailed_filename, 'w+') as f: + tailed_filename = (pathlib.Path.cwd() / pathlib.Path("tailed_file.txt")).resolve() + with open(tailed_filename, "w+") as f: f.write("Preexisting text line 1\n") f.write("Preexisting text line 2\n") @@ -146,13 +148,14 @@ def test_append_to_watched_file_with_absolute_path(): observer = Observer() - sft = SingleFileTail(tailed_filename, append_to_list_partial, - observer=observer, logger=logger) + sft = SingleFileTail( + tailed_filename, append_to_list_partial, observer=observer, logger=logger + ) observer.start() time.sleep(WAIT_TIME) - with open(tailed_filename, 'a+') as f: + with open(tailed_filename, "a+") as f: f.write("Added line 1\n") time.sleep(WAIT_TIME) @@ -168,8 +171,8 @@ def test_append_to_watched_file_with_absolute_path(): def test_append_to_watched_file_with_relative_path(): - tailed_filename = pathlib.Path('tailed_file.txt') - with open(tailed_filename, 'w+') as f: + tailed_filename = pathlib.Path("tailed_file.txt") + with open(tailed_filename, "w+") as f: f.write("Preexisting text line 1\n") f.write("Preexisting text line 2\n") @@ -178,13 +181,14 @@ def test_append_to_watched_file_with_relative_path(): observer = Observer() - sft = SingleFileTail(tailed_filename, append_to_list_partial, - observer=observer, logger=logger) + sft = SingleFileTail( + tailed_filename, append_to_list_partial, observer=observer, logger=logger + ) observer.start() time.sleep(WAIT_TIME) - with open(tailed_filename, 'a+') as f: + with open(tailed_filename, "a+") as f: f.write("Added line 1\n") time.sleep(WAIT_TIME) @@ -200,8 +204,8 @@ def test_append_to_watched_file_with_relative_path(): def test_append_to_watched_file_observer_start_first(): - tailed_filename = pathlib.Path('tailed_file.txt') - with open(tailed_filename, 'w+') as f: + tailed_filename = pathlib.Path("tailed_file.txt") + with open(tailed_filename, "w+") as f: f.write("Preexisting text line 1\n") f.write("Preexisting text line 2\n") @@ -211,10 +215,11 @@ def test_append_to_watched_file_observer_start_first(): observer = Observer() observer.start() - sft = SingleFileTail(tailed_filename, append_to_list_partial, - observer=observer, logger=logger) + sft = SingleFileTail( + tailed_filename, append_to_list_partial, observer=observer, logger=logger + ) - with open(tailed_filename, 'a+') as f: + with open(tailed_filename, "a+") as f: f.write("Added line 1\n") time.sleep(WAIT_TIME) @@ -230,10 +235,10 @@ def test_append_to_watched_file_observer_start_first(): def test_not_watched_file(): - tailed_filename = 'tailed_file.txt' - not_tailed_filename = 'not_tailed_file.txt' - new_not_tailed_filename = not_tailed_filename.replace('.txt', '_moved.txt') - with open(tailed_filename, 'w+') as f: + tailed_filename = "tailed_file.txt" + not_tailed_filename = "not_tailed_file.txt" + new_not_tailed_filename = not_tailed_filename.replace(".txt", "_moved.txt") + with open(tailed_filename, "w+") as f: f.write("Preexisting text line 1\n") f.write("Preexisting text line 2\n") @@ -242,12 +247,13 @@ def test_not_watched_file(): observer = Observer() - sft = SingleFileTail(tailed_filename, append_to_list_partial, - observer=observer, logger=logger) + sft = SingleFileTail( + tailed_filename, append_to_list_partial, observer=observer, logger=logger + ) observer.start() - with open(tailed_filename, 'a+') as f: + with open(tailed_filename, "a+") as f: f.write("Added line 1\n") time.sleep(WAIT_TIME) @@ -255,7 +261,7 @@ def test_not_watched_file(): "Added line 1", ] - with open(not_tailed_filename, 'a+') as f: + with open(not_tailed_filename, "a+") as f: f.write("Added line 1 - not tailed\n") time.sleep(WAIT_TIME) @@ -284,7 +290,7 @@ def test_not_watched_file(): def test_watch_nonexistent_file(): - tailed_filename = 'tailed_file.txt' + tailed_filename = "tailed_file.txt" if os.path.exists(tailed_filename): os.unlink(tailed_filename) @@ -294,8 +300,9 @@ def test_watch_nonexistent_file(): observer = Observer() - sft = SingleFileTail(tailed_filename, append_to_list_partial, - observer=observer, logger=logger) + sft = SingleFileTail( + tailed_filename, append_to_list_partial, observer=observer, logger=logger + ) observer.start() @@ -303,7 +310,7 @@ def test_watch_nonexistent_file(): assert not os.path.exists(tailed_filename) - with open(tailed_filename, 'w+') as f: + with open(tailed_filename, "w+") as f: f.write("Added line 1\n") time.sleep(WAIT_TIME) @@ -322,15 +329,15 @@ def test_watch_nonexistent_file(): def test_follow_watched_file_moved(): - tailed_filename = 'tailed_file_to_move.txt' - new_filename = tailed_filename.replace('_to_move.txt', '_moved.txt') + tailed_filename = "tailed_file_to_move.txt" + new_filename = tailed_filename.replace("_to_move.txt", "_moved.txt") if os.path.exists(new_filename): os.unlink(new_filename) if os.path.exists(tailed_filename): os.unlink(tailed_filename) - with open(tailed_filename, 'w+') as f: + with open(tailed_filename, "w+") as f: f.write("Preexisting text line 1\n") f.write("Preexisting text line 2\n") @@ -339,12 +346,17 @@ def test_follow_watched_file_moved(): observer = Observer() - sft = SingleFileTail(tailed_filename, append_to_list_partial, follow=True, - observer=observer, logger=logger) + sft = SingleFileTail( + tailed_filename, + append_to_list_partial, + follow=True, + observer=observer, + logger=logger, + ) observer.start() - with open(tailed_filename, 'a+') as f: + with open(tailed_filename, "a+") as f: f.write("Added line 1\n") time.sleep(WAIT_TIME) @@ -352,7 +364,7 @@ def test_follow_watched_file_moved(): "Added line 1", ] - with open(tailed_filename, 'a+') as f: + with open(tailed_filename, "a+") as f: f.write("Added line 2") # No newline time.sleep(WAIT_TIME) @@ -367,7 +379,7 @@ def test_follow_watched_file_moved(): "Added line 1", ] - with open(new_filename, 'a+') as f: + with open(new_filename, "a+") as f: f.write(" - end of line 2\n") time.sleep(WAIT_TIME) @@ -376,7 +388,7 @@ def test_follow_watched_file_moved(): "Added line 2 - end of line 2", ] - with open(tailed_filename, 'w+') as f: + with open(tailed_filename, "w+") as f: f.write("New file - text line 1\n") f.write("New file - text line 2\n") time.sleep(WAIT_TIME) @@ -395,15 +407,15 @@ def test_follow_watched_file_moved(): def test_not_followed_watched_file_moved(): - tailed_filename = 'tailed_file_to_move.txt' - new_filename = tailed_filename.replace('_to_move.txt', '_moved.txt') + tailed_filename = "tailed_file_to_move.txt" + new_filename = tailed_filename.replace("_to_move.txt", "_moved.txt") if os.path.exists(new_filename): os.unlink(new_filename) if os.path.exists(tailed_filename): os.unlink(tailed_filename) - with open(tailed_filename, 'w+') as f: + with open(tailed_filename, "w+") as f: f.write("Preexisting text line 1\n") f.write("Preexisting text line 2\n") @@ -412,12 +424,17 @@ def test_not_followed_watched_file_moved(): observer = Observer() - sft = SingleFileTail(tailed_filename, append_to_list_partial, follow=False, - observer=observer, logger=logger) + sft = SingleFileTail( + tailed_filename, + append_to_list_partial, + follow=False, + observer=observer, + logger=logger, + ) observer.start() - with open(tailed_filename, 'a+') as f: + with open(tailed_filename, "a+") as f: f.write("Added line 1\n") time.sleep(WAIT_TIME) @@ -425,7 +442,7 @@ def test_not_followed_watched_file_moved(): "Added line 1", ] - with open(tailed_filename, 'a+') as f: + with open(tailed_filename, "a+") as f: f.write("Added line 2") # No newline time.sleep(WAIT_TIME) @@ -441,7 +458,7 @@ def test_not_followed_watched_file_moved(): "Added line 2", ] - with open(new_filename, 'a+') as f: + with open(new_filename, "a+") as f: f.write(" - end of line 2\n") time.sleep(WAIT_TIME) @@ -452,7 +469,7 @@ def test_not_followed_watched_file_moved(): assert not os.path.exists(tailed_filename) - with open(tailed_filename, 'w+') as f: + with open(tailed_filename, "w+") as f: f.write("Recreated file - text line 1\n") f.write("Recreated file - text line 2\n") time.sleep(WAIT_TIME) @@ -475,9 +492,9 @@ def test_not_followed_watched_file_moved(): def test_non_watched_file_moved(): - tailed_filename = 'tailed_file_to_move.txt' - not_tailed_filename = f'not_{tailed_filename}' - new_not_tailed_filename = not_tailed_filename.replace('_to_move.txt', '_moved.txt') + tailed_filename = "tailed_file_to_move.txt" + not_tailed_filename = f"not_{tailed_filename}" + new_not_tailed_filename = not_tailed_filename.replace("_to_move.txt", "_moved.txt") if os.path.exists(not_tailed_filename): os.unlink(not_tailed_filename) @@ -486,10 +503,10 @@ def test_non_watched_file_moved(): if os.path.exists(tailed_filename): os.unlink(tailed_filename) - with open(not_tailed_filename, 'w+') as f: + with open(not_tailed_filename, "w+") as f: f.write("Text here will not be monitored\n") - with open(tailed_filename, 'w+') as f: + with open(tailed_filename, "w+") as f: f.write("Preexisting text line 1\n") f.write("Preexisting text line 2\n") @@ -498,12 +515,13 @@ def test_non_watched_file_moved(): observer = Observer() - sft = SingleFileTail(tailed_filename, append_to_list_partial, - observer=observer, logger=logger) + sft = SingleFileTail( + tailed_filename, append_to_list_partial, observer=observer, logger=logger + ) observer.start() - with open(tailed_filename, 'a+') as f: + with open(tailed_filename, "a+") as f: f.write("Added line 1\n") time.sleep(WAIT_TIME) @@ -527,8 +545,8 @@ def test_non_watched_file_moved(): def test_watched_file_deleted(): - tailed_filename = 'tailed_file_deleted.txt' - with open(tailed_filename, 'w+') as f: + tailed_filename = "tailed_file_deleted.txt" + with open(tailed_filename, "w+") as f: f.write("Preexisting text line 1\n") f.write("Preexisting text line 2\n") @@ -537,13 +555,14 @@ def test_watched_file_deleted(): observer = Observer() - sft = SingleFileTail(tailed_filename, append_to_list_partial, - observer=observer, logger=logger) + sft = SingleFileTail( + tailed_filename, append_to_list_partial, observer=observer, logger=logger + ) observer.start() time.sleep(WAIT_TIME) - with open(tailed_filename, 'a+') as f: + with open(tailed_filename, "a+") as f: f.write("Added line 1\n") time.sleep(WAIT_TIME) @@ -564,8 +583,8 @@ def test_watched_file_deleted(): def test_watched_file_immediately_deleted(): - tailed_filename = 'tailed_file_deleted.txt' - with open(tailed_filename, 'w+') as f: + tailed_filename = "tailed_file_deleted.txt" + with open(tailed_filename, "w+") as f: f.write("Preexisting text line 1\n") f.write("Preexisting text line 2\n") @@ -574,8 +593,9 @@ def test_watched_file_immediately_deleted(): observer = Observer() - sft = SingleFileTail(tailed_filename, append_to_list_partial, - observer=observer, logger=logger) + sft = SingleFileTail( + tailed_filename, append_to_list_partial, observer=observer, logger=logger + ) observer.start() time.sleep(WAIT_TIME) @@ -589,14 +609,14 @@ def test_watched_file_immediately_deleted(): observer.stop() -if __name__ == '__main__': +if __name__ == "__main__": import sys logger.setLevel(logging.DEBUG) handler = logging.StreamHandler(sys.stderr) handler.setLevel(logging.DEBUG) - formatter = logging.Formatter('%(name)s: %(levelname)s: %(message)s') + formatter = logging.Formatter("%(name)s: %(levelname)s: %(message)s") logger.addHandler(handler) diff --git a/contrib/linux/tests/test_tail_manager.py b/contrib/linux/tests/test_tail_manager.py index 6664c59b92..722167634a 100644 --- a/contrib/linux/tests/test_tail_manager.py +++ b/contrib/linux/tests/test_tail_manager.py @@ -31,6 +31,7 @@ # Helper function + def append_to_list(list_to_append, path, element): logger.debug(f"Appending ({path}):\n{element} to {list_to_append}") list_to_append.append(element) @@ -41,17 +42,18 @@ def test_initialized_without_logger(): TailManager() except Exception as e: expected_message = "TailManager was initialized without a logger" - exc_msg = getattr(e, 'message', e.args[0]) + exc_msg = getattr(e, "message", e.args[0]) if exc_msg != expected_message: raise e else: - raise AssertionError("TailManager initialized fine without a " - "logger parameter") + raise AssertionError( + "TailManager initialized fine without a " "logger parameter" + ) def test_append_to_watched_file_with_absolute_path(): - tailed_filename = (pathlib.Path.cwd() / pathlib.Path('tailed_file.txt')).resolve() - with open(tailed_filename, 'w+') as f: + tailed_filename = (pathlib.Path.cwd() / pathlib.Path("tailed_file.txt")).resolve() + with open(tailed_filename, "w+") as f: f.write("Preexisting text line 1\n") f.write("Preexisting text line 2\n") @@ -62,7 +64,7 @@ def test_append_to_watched_file_with_absolute_path(): tm.tail_file(tailed_filename, handler=append_to_list_partial) tm.start() - with open(tailed_filename, 'a+') as f: + with open(tailed_filename, "a+") as f: f.write("Added line 1\n") time.sleep(WAIT_TIME) @@ -78,10 +80,10 @@ def test_append_to_watched_file_with_absolute_path(): def test_not_watched_file(): - tailed_filename = 'tailed_file.txt' - not_tailed_filename = 'not_tailed_file.txt' - new_not_tailed_filename = not_tailed_filename.replace('.txt', '_moved.txt') - with open(tailed_filename, 'w+') as f: + tailed_filename = "tailed_file.txt" + not_tailed_filename = "not_tailed_file.txt" + new_not_tailed_filename = not_tailed_filename.replace(".txt", "_moved.txt") + with open(tailed_filename, "w+") as f: f.write("Preexisting text line 1\n") f.write("Preexisting text line 2\n") @@ -92,7 +94,7 @@ def test_not_watched_file(): tm.tail_file(tailed_filename, handler=append_to_list_partial) tm.start() - with open(tailed_filename, 'a+') as f: + with open(tailed_filename, "a+") as f: f.write("Added line 1\n") time.sleep(WAIT_TIME) @@ -100,7 +102,7 @@ def test_not_watched_file(): "Added line 1", ] - with open(not_tailed_filename, 'a+') as f: + with open(not_tailed_filename, "a+") as f: f.write("Added line 1 - not tailed\n") time.sleep(WAIT_TIME) @@ -129,7 +131,7 @@ def test_not_watched_file(): def test_watch_nonexistent_file(): - tailed_filename = 'tailed_file.txt' + tailed_filename = "tailed_file.txt" if os.path.exists(tailed_filename): os.unlink(tailed_filename) @@ -144,7 +146,7 @@ def test_watch_nonexistent_file(): assert appended_lines == [] - with open(tailed_filename, 'w+') as f: + with open(tailed_filename, "w+") as f: f.write("Added line 1\n") time.sleep(WAIT_TIME) @@ -161,15 +163,15 @@ def test_watch_nonexistent_file(): def test_follow_watched_file_moved(): - tailed_filename = 'tailed_file_to_move.txt' - new_filename = tailed_filename.replace('_to_move.txt', '_moved.txt') + tailed_filename = "tailed_file_to_move.txt" + new_filename = tailed_filename.replace("_to_move.txt", "_moved.txt") if os.path.exists(new_filename): os.unlink(new_filename) if os.path.exists(tailed_filename): os.unlink(tailed_filename) - with open(tailed_filename, 'w+') as f: + with open(tailed_filename, "w+") as f: f.write("Preexisting text line 1\n") f.write("Preexisting text line 2\n") @@ -180,7 +182,7 @@ def test_follow_watched_file_moved(): tm.tail_file(tailed_filename, handler=append_to_list_partial, follow=True) tm.start() - with open(tailed_filename, 'a+') as f: + with open(tailed_filename, "a+") as f: f.write("Added line 1\n") time.sleep(WAIT_TIME) @@ -188,7 +190,7 @@ def test_follow_watched_file_moved(): "Added line 1", ] - with open(tailed_filename, 'a+') as f: + with open(tailed_filename, "a+") as f: f.write("Added line 2") # No newline time.sleep(WAIT_TIME) @@ -203,7 +205,7 @@ def test_follow_watched_file_moved(): "Added line 1", ] - with open(new_filename, 'a+') as f: + with open(new_filename, "a+") as f: f.write(" - end of line 2\n") time.sleep(WAIT_TIME) @@ -212,7 +214,7 @@ def test_follow_watched_file_moved(): "Added line 2 - end of line 2", ] - with open(tailed_filename, 'w+') as f: + with open(tailed_filename, "w+") as f: f.write("New file - text line 1\n") f.write("New file - text line 2\n") time.sleep(WAIT_TIME) @@ -231,15 +233,15 @@ def test_follow_watched_file_moved(): def test_not_followed_watched_file_moved(): - tailed_filename = 'tailed_file_to_move.txt' - new_filename = tailed_filename.replace('_to_move.txt', '_moved.txt') + tailed_filename = "tailed_file_to_move.txt" + new_filename = tailed_filename.replace("_to_move.txt", "_moved.txt") if os.path.exists(new_filename): os.unlink(new_filename) if os.path.exists(tailed_filename): os.unlink(tailed_filename) - with open(tailed_filename, 'w+') as f: + with open(tailed_filename, "w+") as f: f.write("Preexisting text line 1\n") f.write("Preexisting text line 2\n") @@ -250,7 +252,7 @@ def test_not_followed_watched_file_moved(): tm.tail_file(tailed_filename, handler=append_to_list_partial, follow=False) tm.start() - with open(tailed_filename, 'a+') as f: + with open(tailed_filename, "a+") as f: f.write("Added line 1\n") time.sleep(WAIT_TIME) @@ -258,7 +260,7 @@ def test_not_followed_watched_file_moved(): "Added line 1", ] - with open(tailed_filename, 'a+') as f: + with open(tailed_filename, "a+") as f: f.write("Added line 2") # No newline time.sleep(WAIT_TIME) @@ -274,7 +276,7 @@ def test_not_followed_watched_file_moved(): "Added line 2", ] - with open(new_filename, 'a+') as f: + with open(new_filename, "a+") as f: f.write(" - end of line 2\n") time.sleep(WAIT_TIME) @@ -283,7 +285,7 @@ def test_not_followed_watched_file_moved(): "Added line 2", ] - with open(tailed_filename, 'w+') as f: + with open(tailed_filename, "w+") as f: f.write("Recreated file - text line 1\n") f.write("Recreated file - text line 2\n") time.sleep(WAIT_TIME) @@ -304,9 +306,9 @@ def test_not_followed_watched_file_moved(): def test_non_watched_file_moved(): - tailed_filename = 'tailed_file_to_move.txt' - not_tailed_filename = f'not_{tailed_filename}' - new_not_tailed_filename = not_tailed_filename.replace('_to_move.txt', '_moved.txt') + tailed_filename = "tailed_file_to_move.txt" + not_tailed_filename = f"not_{tailed_filename}" + new_not_tailed_filename = not_tailed_filename.replace("_to_move.txt", "_moved.txt") if os.path.exists(not_tailed_filename): os.unlink(not_tailed_filename) @@ -315,10 +317,10 @@ def test_non_watched_file_moved(): if os.path.exists(tailed_filename): os.unlink(tailed_filename) - with open(not_tailed_filename, 'w+') as f: + with open(not_tailed_filename, "w+") as f: f.write("Text here will not be monitored\n") - with open(tailed_filename, 'w+') as f: + with open(tailed_filename, "w+") as f: f.write("Preexisting text line 1\n") f.write("Preexisting text line 2\n") @@ -329,7 +331,7 @@ def test_non_watched_file_moved(): tm.tail_file(tailed_filename, handler=append_to_list_partial) tm.start() - with open(tailed_filename, 'a+') as f: + with open(tailed_filename, "a+") as f: f.write("Added line 1\n") time.sleep(WAIT_TIME) @@ -353,8 +355,8 @@ def test_non_watched_file_moved(): def test_watched_file_deleted(): - tailed_filename = 'tailed_file_deleted.txt' - with open(tailed_filename, 'w+') as f: + tailed_filename = "tailed_file_deleted.txt" + with open(tailed_filename, "w+") as f: f.write("Preexisting text line 1\n") f.write("Preexisting text line 2\n") @@ -365,7 +367,7 @@ def test_watched_file_deleted(): tm.tail_file(tailed_filename, handler=append_to_list_partial) tm.start() - with open(tailed_filename, 'a+') as f: + with open(tailed_filename, "a+") as f: f.write("Added line 1\n") time.sleep(WAIT_TIME) @@ -383,8 +385,8 @@ def test_watched_file_deleted(): def test_watched_file_immediately_deleted(): - tailed_filename = 'tailed_file_deleted.txt' - with open(tailed_filename, 'w+') as f: + tailed_filename = "tailed_file_deleted.txt" + with open(tailed_filename, "w+") as f: f.write("Preexisting text line 1\n") f.write("Preexisting text line 2\n") @@ -401,8 +403,8 @@ def test_watched_file_immediately_deleted(): def test_append_to_watched_file(): - tailed_filename = 'tailed_file.txt' - with open(tailed_filename, 'w+') as f: + tailed_filename = "tailed_file.txt" + with open(tailed_filename, "w+") as f: f.write("Preexisting text line 1\n") f.write("Preexisting text line 2\n") @@ -413,7 +415,7 @@ def test_append_to_watched_file(): tm.tail_file(tailed_filename, handler=append_to_list_partial) tm.start() - with open(tailed_filename, 'a+') as f: + with open(tailed_filename, "a+") as f: f.write("Added line 1\n") time.sleep(WAIT_TIME) @@ -421,7 +423,7 @@ def test_append_to_watched_file(): "Added line 1", ] - with open(tailed_filename, 'a+') as f: + with open(tailed_filename, "a+") as f: f.write("Added line 2\n") time.sleep(WAIT_TIME) @@ -430,7 +432,7 @@ def test_append_to_watched_file(): "Added line 2", ] - with open(tailed_filename, 'a+') as f: + with open(tailed_filename, "a+") as f: f.write("Start of added partial line 1") time.sleep(WAIT_TIME) @@ -439,7 +441,7 @@ def test_append_to_watched_file(): "Added line 2", ] - with open(tailed_filename, 'a+') as f: + with open(tailed_filename, "a+") as f: f.write(" - finished partial line 1\nStart of added partial line 2") time.sleep(WAIT_TIME) @@ -449,7 +451,7 @@ def test_append_to_watched_file(): "Start of added partial line 1 - finished partial line 1", ] - with open(tailed_filename, 'a+') as f: + with open(tailed_filename, "a+") as f: f.write(" - finished partial line 2\n") time.sleep(WAIT_TIME) @@ -460,7 +462,7 @@ def test_append_to_watched_file(): "Start of added partial line 2 - finished partial line 2", ] - with open(tailed_filename, 'a+') as f: + with open(tailed_filename, "a+") as f: f.write("Final line without a newline") time.sleep(WAIT_TIME) @@ -481,8 +483,8 @@ def test_append_to_watched_file(): def test_tail_file_twice(): - tailed_filename = 'tailed_file.txt' - with open(tailed_filename, 'w+') as f: + tailed_filename = "tailed_file.txt" + with open(tailed_filename, "w+") as f: f.write("Preexisting text line 1\n") f.write("Preexisting text line 2\n") @@ -494,7 +496,7 @@ def test_tail_file_twice(): tm.tail_file(tailed_filename, handler=append_to_list_partial) tm.start() - with open(tailed_filename, 'a+') as f: + with open(tailed_filename, "a+") as f: f.write("Added line 1\n") time.sleep(WAIT_TIME) @@ -502,7 +504,7 @@ def test_tail_file_twice(): "Added line 1", ] - with open(tailed_filename, 'a+') as f: + with open(tailed_filename, "a+") as f: f.write("Added line 2\n") time.sleep(WAIT_TIME) @@ -511,7 +513,7 @@ def test_tail_file_twice(): "Added line 2", ] - with open(tailed_filename, 'a+') as f: + with open(tailed_filename, "a+") as f: f.write("Start of added partial line 1") time.sleep(WAIT_TIME) @@ -520,7 +522,7 @@ def test_tail_file_twice(): "Added line 2", ] - with open(tailed_filename, 'a+') as f: + with open(tailed_filename, "a+") as f: f.write(" - finished partial line 1\nStart of added partial line 2") time.sleep(WAIT_TIME) @@ -530,7 +532,7 @@ def test_tail_file_twice(): "Start of added partial line 1 - finished partial line 1", ] - with open(tailed_filename, 'a+') as f: + with open(tailed_filename, "a+") as f: f.write(" - finished partial line 2\n") time.sleep(WAIT_TIME) @@ -541,7 +543,7 @@ def test_tail_file_twice(): "Start of added partial line 2 - finished partial line 2", ] - with open(tailed_filename, 'a+') as f: + with open(tailed_filename, "a+") as f: f.write("Final line without a newline") time.sleep(WAIT_TIME) @@ -562,8 +564,8 @@ def test_tail_file_twice(): def test_stop(): - tailed_filename = 'tailed_file_stop.txt' - with open(tailed_filename, 'w+') as f: + tailed_filename = "tailed_file_stop.txt" + with open(tailed_filename, "w+") as f: f.write("Preexisting text line 1\n") f.write("Preexisting text line 2\n") @@ -574,7 +576,7 @@ def test_stop(): tm.tail_file(tailed_filename, handler=append_to_list_partial) tm.start() - with open(tailed_filename, 'a+') as f: + with open(tailed_filename, "a+") as f: f.write("Added line 1\n") time.sleep(WAIT_TIME) @@ -582,7 +584,7 @@ def test_stop(): "Added line 1", ] - with open(tailed_filename, 'a+') as f: + with open(tailed_filename, "a+") as f: f.write("Final line without a newline") time.sleep(WAIT_TIME) @@ -598,8 +600,8 @@ def test_stop(): def test_stop_twice(): - tailed_filename = 'tailed_file_stop.txt' - with open(tailed_filename, 'w+') as f: + tailed_filename = "tailed_file_stop.txt" + with open(tailed_filename, "w+") as f: f.write("Preexisting text line 1\n") f.write("Preexisting text line 2\n") @@ -610,7 +612,7 @@ def test_stop_twice(): tm.tail_file(tailed_filename, handler=append_to_list_partial) tm.start() - with open(tailed_filename, 'a+') as f: + with open(tailed_filename, "a+") as f: f.write("Added line 1\n") time.sleep(WAIT_TIME) @@ -618,7 +620,7 @@ def test_stop_twice(): "Added line 1", ] - with open(tailed_filename, 'a+') as f: + with open(tailed_filename, "a+") as f: f.write("Final line without a newline") time.sleep(WAIT_TIME) @@ -634,14 +636,14 @@ def test_stop_twice(): os.unlink(tailed_filename) -if __name__ == '__main__': +if __name__ == "__main__": import sys logger.setLevel(logging.DEBUG) handler = logging.StreamHandler(sys.stderr) handler.setLevel(logging.DEBUG) - formatter = logging.Formatter('%(name)s: %(levelname)s: %(message)s') + formatter = logging.Formatter("%(name)s: %(levelname)s: %(message)s") logger.addHandler(handler) From 1e5e389e09a645763cd06c770566125b916f95fc Mon Sep 17 00:00:00 2001 From: blag Date: Mon, 12 Apr 2021 13:23:28 -0700 Subject: [PATCH 24/35] Make flake8 happy --- contrib/linux/sensors/file_watch_sensor.py | 2 +- contrib/linux/tests/test_file_watch_sensor.py | 11 +++++------ contrib/linux/tests/test_tail_manager.py | 2 -- 3 files changed, 6 insertions(+), 9 deletions(-) diff --git a/contrib/linux/sensors/file_watch_sensor.py b/contrib/linux/sensors/file_watch_sensor.py index bdf7594208..e9bddebf50 100644 --- a/contrib/linux/sensors/file_watch_sensor.py +++ b/contrib/linux/sensors/file_watch_sensor.py @@ -347,7 +347,7 @@ def close(self, event=None, emit_remaining=True, end_parent_watch=True): self.logger.debug(f"Closing file handle {self.fd}") os.close(self.fd) self.fd = None - self.logger.debug(f"Closed file handle") + self.logger.debug("Closed file handle") class TailManager(object): diff --git a/contrib/linux/tests/test_file_watch_sensor.py b/contrib/linux/tests/test_file_watch_sensor.py index 08edf71465..f28b5776c7 100644 --- a/contrib/linux/tests/test_file_watch_sensor.py +++ b/contrib/linux/tests/test_file_watch_sensor.py @@ -14,12 +14,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -import functools import logging import os import pathlib -import sys -import threading +# import sys +# import threading import time import eventlet @@ -49,7 +48,7 @@ def test_file_watch_sensor(): time.sleep(WAIT_TIME) - # th = threading.Thread(target=fws.run) + th = threading.Thread(target=fws.run) th = eventlet.spawn(fws.run) # th.start() @@ -127,7 +126,7 @@ def test_file_watch_sensor_without_trigger_filepath(): time.sleep(WAIT_TIME) - # th = threading.Thread(target=fws.run) + th = threading.Thread(target=fws.run) th = eventlet.spawn(fws.run) # th.start() @@ -167,7 +166,7 @@ def test_file_watch_sensor_without_trigger_ref(): time.sleep(WAIT_TIME) - # th = threading.Thread(target=fws.run) + th = threading.Thread(target=fws.run) th = eventlet.spawn(fws.run) # th.start() diff --git a/contrib/linux/tests/test_tail_manager.py b/contrib/linux/tests/test_tail_manager.py index 722167634a..b74a886a88 100644 --- a/contrib/linux/tests/test_tail_manager.py +++ b/contrib/linux/tests/test_tail_manager.py @@ -20,8 +20,6 @@ import pathlib import time -from watchdog.observers import Observer - from file_watch_sensor import TailManager WAIT_TIME = 1 From 5e0b7e6a81932dd455daf25ea018b2d40a43ee40 Mon Sep 17 00:00:00 2001 From: blag Date: Mon, 12 Apr 2021 13:25:18 -0700 Subject: [PATCH 25/35] Make pylint happy --- contrib/linux/tests/test_single_file_tail.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/contrib/linux/tests/test_single_file_tail.py b/contrib/linux/tests/test_single_file_tail.py index 1b3aeffe16..c8d573e717 100644 --- a/contrib/linux/tests/test_single_file_tail.py +++ b/contrib/linux/tests/test_single_file_tail.py @@ -129,7 +129,8 @@ def test_initialize_without_logger(): SingleFileTail(None, None, fd=None) except Exception as e: expected_message = "SingleFileTail was initialized without a logger" - if hasattr(e, "message") and e.message != expected_message: + exception_message = getattr(e, "message", e.args[0]) + if exception_message != expected_message: raise e else: raise AssertionError( From d90be9cf7472e9c1ec950f8eb4842d8efca0b810 Mon Sep 17 00:00:00 2001 From: blag Date: Mon, 12 Apr 2021 13:28:38 -0700 Subject: [PATCH 26/35] Make black happy --- contrib/linux/tests/test_file_watch_sensor.py | 1 + 1 file changed, 1 insertion(+) diff --git a/contrib/linux/tests/test_file_watch_sensor.py b/contrib/linux/tests/test_file_watch_sensor.py index f28b5776c7..ffe52d27b9 100644 --- a/contrib/linux/tests/test_file_watch_sensor.py +++ b/contrib/linux/tests/test_file_watch_sensor.py @@ -17,6 +17,7 @@ import logging import os import pathlib + # import sys # import threading import time From 87d32aa4322ead7ca26a8838b94a2adcac012341 Mon Sep 17 00:00:00 2001 From: blag Date: Mon, 12 Apr 2021 13:33:39 -0700 Subject: [PATCH 27/35] Don't use threading.Thread() after all --- contrib/linux/tests/test_file_watch_sensor.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/contrib/linux/tests/test_file_watch_sensor.py b/contrib/linux/tests/test_file_watch_sensor.py index ffe52d27b9..e85c2fb4c7 100644 --- a/contrib/linux/tests/test_file_watch_sensor.py +++ b/contrib/linux/tests/test_file_watch_sensor.py @@ -49,7 +49,7 @@ def test_file_watch_sensor(): time.sleep(WAIT_TIME) - th = threading.Thread(target=fws.run) + # th = threading.Thread(target=fws.run) th = eventlet.spawn(fws.run) # th.start() @@ -127,7 +127,7 @@ def test_file_watch_sensor_without_trigger_filepath(): time.sleep(WAIT_TIME) - th = threading.Thread(target=fws.run) + # th = threading.Thread(target=fws.run) th = eventlet.spawn(fws.run) # th.start() @@ -167,7 +167,7 @@ def test_file_watch_sensor_without_trigger_ref(): time.sleep(WAIT_TIME) - th = threading.Thread(target=fws.run) + # th = threading.Thread(target=fws.run) th = eventlet.spawn(fws.run) # th.start() From e64e5472fc1625228f6ab28a73f0f58550b10642 Mon Sep 17 00:00:00 2001 From: blag Date: Mon, 12 Apr 2021 13:38:34 -0700 Subject: [PATCH 28/35] Do whatever it takes to make all of the linters happy --- contrib/linux/tests/test_file_watch_sensor.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/contrib/linux/tests/test_file_watch_sensor.py b/contrib/linux/tests/test_file_watch_sensor.py index e85c2fb4c7..a0f210b524 100644 --- a/contrib/linux/tests/test_file_watch_sensor.py +++ b/contrib/linux/tests/test_file_watch_sensor.py @@ -51,7 +51,7 @@ def test_file_watch_sensor(): # th = threading.Thread(target=fws.run) th = eventlet.spawn(fws.run) - # th.start() + th.start() time.sleep(WAIT_TIME) @@ -129,7 +129,7 @@ def test_file_watch_sensor_without_trigger_filepath(): # th = threading.Thread(target=fws.run) th = eventlet.spawn(fws.run) - # th.start() + th.start() time.sleep(WAIT_TIME) @@ -169,7 +169,7 @@ def test_file_watch_sensor_without_trigger_ref(): # th = threading.Thread(target=fws.run) th = eventlet.spawn(fws.run) - # th.start() + th.start() time.sleep(WAIT_TIME) From 3055413aa05318416986b5d0e6c7f9eaab9d5a11 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Wed, 6 Oct 2021 19:17:09 -0500 Subject: [PATCH 29/35] add changelog entry --- CHANGELOG.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 12a169adcc..e2ee275864 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -74,6 +74,15 @@ Changed * Silence pylint about dev/debugging utility (tools/direct_queue_publisher.py) that uses pika because kombu doesn't support what it does. If anyone uses that utility, they have to install pika manually. #5380 +* The FileWatchSensor in the linux pack uses `watchdog` now instead of `logshipper` (`logshipper` and its requirement `pyinotify` are unmaintained). + The `watchdog` project is actively maintained, and has wide support for Linux, MacOS, BSD, Windows, and a polling fallback implementation. + Dropping `pyinotify` has a side benefit of allowing MacOS users to more easily hack on StackStorm's code, since `pyinotify` cannot install on MacOS. + + The FileWatchSensor is now an excellent example of how to write a sensor following this refactor. It breaks up the code into well structured classes + that all have a single focus. You can also run the sensor Python script itself, which makes development and testing much easier. #5096 + + Contributed by @blag + Fixed ~~~~~ From 83fa1ec00c5c0340684d19eeca1eebceba8f3c16 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Fri, 19 Jan 2024 18:43:25 -0600 Subject: [PATCH 30/35] Fix borked merge --- contrib/linux/sensors/file_watch_sensor.py | 4 ++-- st2actions/requirements.txt | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/contrib/linux/sensors/file_watch_sensor.py b/contrib/linux/sensors/file_watch_sensor.py index b54c7ca15a..10b8623fd3 100644 --- a/contrib/linux/sensors/file_watch_sensor.py +++ b/contrib/linux/sensors/file_watch_sensor.py @@ -446,7 +446,7 @@ def remove_trigger(self, trigger): file_path = trigger["parameters"].get("file_path", None) if not file_path: - self.log.error('Received trigger type without "file_path" field.') + self.log.error("Received trigger type without 'file_path' field.") return self.tail_manager.stop_tailing_file(file_path, self._handle_line) @@ -470,7 +470,7 @@ def _handle_line(self, file_path, line): self.log.debug( f"Sending payload {payload} for trigger {trigger} to sensor_service." ) - self.sensor_service.dispatch(trigger=self.trigger, payload=payload) + self.sensor_service.dispatch(trigger=trigger, payload=payload) if __name__ == "__main__": diff --git a/st2actions/requirements.txt b/st2actions/requirements.txt index 041ed3a0d9..4d2ad95273 100644 --- a/st2actions/requirements.txt +++ b/st2actions/requirements.txt @@ -13,7 +13,6 @@ gitpython<=3.1.37 jinja2==2.11.3 kombu==5.0.2 lockfile==0.12.2 -logshipper@ git+https://github.com/StackStorm/logshipper.git@stackstorm_patched ; platform_system=="Linux" oslo.config>=1.12.1,<1.13 oslo.utils<5.0,>=4.0.0 pyparsing<3 From bab2952c51e3943b7e911b62ff7b22abbea0383a Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Fri, 19 Jan 2024 19:05:54 -0600 Subject: [PATCH 31/35] pin transitive dep in test-requirements.txt --- test-requirements.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test-requirements.txt b/test-requirements.txt index adf875e0cf..2f4935c290 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -15,6 +15,8 @@ tabulate unittest2 sphinx==1.7.9 sphinx-autobuild +# pin transitive dep that as newer versions of alabaster do not support older sphinx versions +alabaster<0.7.14 ; python_version >= '3.9' # nosetests enhancements rednose nose-timer==1.0.1 From 5b381069aa26acfb6373f2cd0278db2b53cc18c4 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Mon, 29 Jan 2024 16:47:09 -0600 Subject: [PATCH 32/35] regen st2 lockfile to switch logshipper -> watchdog Lockfile diff: lockfiles/st2.lock [st2] == Added dependencies == watchdog 3.0.0 == Removed dependencies == logshipper 0.0.0 pyinotify 0.9.6 --- lockfiles/st2.lock | 132 +++++++++++++++++++++++++++++++-------------- 1 file changed, 92 insertions(+), 40 deletions(-) diff --git a/lockfiles/st2.lock b/lockfiles/st2.lock index 0d618ae884..005676efab 100644 --- a/lockfiles/st2.lock +++ b/lockfiles/st2.lock @@ -29,7 +29,6 @@ // "jsonschema<4,>=3", // "kombu", // "lockfile", -// "logshipper@ git+https://github.com/StackStorm/logshipper.git@stackstorm_patched ; platform_system == \"Linux\"", // "mail-parser==3.15.0", // "mock", // "mongoengine", @@ -44,7 +43,6 @@ // "prettytable", // "prompt-toolkit<2", // "psutil", -// "pyinotify<=0.10,>=0.9.5; platform_system == \"Linux\"", // "pymongo", // "pyrabbit", // "pysocks", @@ -74,6 +72,7 @@ // "ujson", // "unittest2", // "virtualenv", +// "watchdog", // "webob", // "webtest", // "wheel", @@ -1693,29 +1692,6 @@ "requires_python": null, "version": "0.12.2" }, - { - "artifacts": [ - { - "algorithm": "sha256", - "hash": "b0fb343d27ee21201dc0002bd48cd7591dfe3cb17914b56326f99344bfb89dc9", - "url": "git+https://github.com/StackStorm/logshipper.git@stackstorm_patched" - } - ], - "project_name": "logshipper", - "requires_dists": [ - "eventlet", - "pika", - "pyinotify", - "python-dateutil", - "python-statsd", - "pytz", - "pyyaml", - "requests", - "six" - ], - "requires_python": null, - "version": "0.0.0" - }, { "artifacts": [ { @@ -2724,19 +2700,6 @@ "requires_python": "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,>=2.7", "version": "2.21" }, - { - "artifacts": [ - { - "algorithm": "sha256", - "hash": "9c998a5d7606ca835065cdabc013ae6c66eb9ea76a00a1e3bc6e0cfe2b4f71f4", - "url": "https://files.pythonhosted.org/packages/e3/c0/fd5b18dde17c1249658521f69598f3252f11d9d7a980c5be8619970646e1/pyinotify-0.9.6.tar.gz" - } - ], - "project_name": "pyinotify", - "requires_dists": [], - "requires_python": null, - "version": "0.9.6" - }, { "artifacts": [ { @@ -4452,6 +4415,96 @@ "requires_python": ">=3.7.0", "version": "2.1.2" }, + { + "artifacts": [ + { + "algorithm": "sha256", + "hash": "d429c2430c93b7903914e4db9a966c7f2b068dd2ebdd2fa9b9ce094c7d459f33", + "url": "https://files.pythonhosted.org/packages/2b/f0/456948b865ab259784f774154e7d65844fa9757522fdb11533fbf8ae7aca/watchdog-3.0.0-py3-none-manylinux2014_x86_64.whl" + }, + { + "algorithm": "sha256", + "hash": "c07253088265c363d1ddf4b3cdb808d59a0468ecd017770ed716991620b8f77a", + "url": "https://files.pythonhosted.org/packages/21/72/46fd174352cd88b9157ade77e3b8835125d4b1e5186fc7f1e8c44664e029/watchdog-3.0.0-py3-none-manylinux2014_i686.whl" + }, + { + "algorithm": "sha256", + "hash": "25f70b4aa53bd743729c7475d7ec41093a580528b100e9a8c5b5efe8899592fc", + "url": "https://files.pythonhosted.org/packages/2e/54/48527f3aea4f7ed331072352fee034a7f3d6ec7a2ed873681738b2586498/watchdog-3.0.0-cp38-cp38-macosx_10_9_x86_64.whl" + }, + { + "algorithm": "sha256", + "hash": "c9d8c8ec7efb887333cf71e328e39cffbf771d8f8f95d308ea4125bf5f90ba64", + "url": "https://files.pythonhosted.org/packages/30/65/9e36a3c821d47a22e54a8fc73681586b2d26e82d24ea3af63acf2ef78f97/watchdog-3.0.0-pp39-pypy39_pp73-macosx_10_9_x86_64.whl" + }, + { + "algorithm": "sha256", + "hash": "ba07e92756c97e3aca0912b5cbc4e5ad802f4557212788e72a72a47ff376950d", + "url": "https://files.pythonhosted.org/packages/40/1b/4e6d3e0f587587931f590531b4ed08070d71a9efb35541d792a68d8ee593/watchdog-3.0.0-py3-none-manylinux2014_s390x.whl" + }, + { + "algorithm": "sha256", + "hash": "233b5817932685d39a7896b1090353fc8efc1ef99c9c054e46c8002561252fb8", + "url": "https://files.pythonhosted.org/packages/51/b9/444a984b1667013bac41b31b45d9718e069cc7502a43a924896806605d83/watchdog-3.0.0-cp39-cp39-macosx_11_0_arm64.whl" + }, + { + "algorithm": "sha256", + "hash": "51f90f73b4697bac9c9a78394c3acbbd331ccd3655c11be1a15ae6fe289a8c83", + "url": "https://files.pythonhosted.org/packages/71/3a/b12740f4f60861240d57b42a2ac6ac0a2821db506c4435f7872c1fad867d/watchdog-3.0.0-py3-none-manylinux2014_ppc64le.whl" + }, + { + "algorithm": "sha256", + "hash": "5113334cf8cf0ac8cd45e1f8309a603291b614191c9add34d33075727a967709", + "url": "https://files.pythonhosted.org/packages/74/3c/e4b77f4f069aca2b6e35925db7a1aa6cb600dcb52fc3e962284640ca37f3/watchdog-3.0.0-py3-none-manylinux2014_ppc64.whl" + }, + { + "algorithm": "sha256", + "hash": "7c5f84b5194c24dd573fa6472685b2a27cc5a17fe5f7b6fd40345378ca6812e3", + "url": "https://files.pythonhosted.org/packages/75/fe/d9a37d8df76878853f68dd665ec6d2c7a984645de460164cb880a93ffe6b/watchdog-3.0.0-cp39-cp39-macosx_10_9_universal2.whl" + }, + { + "algorithm": "sha256", + "hash": "8ae9cda41fa114e28faf86cb137d751a17ffd0316d1c34ccf2235e8a84365c7f", + "url": "https://files.pythonhosted.org/packages/7f/6e/7ca8ed16928d7b11da69372f55c64a09dce649d2b24b03f7063cd8683c4b/watchdog-3.0.0-cp38-cp38-macosx_10_9_universal2.whl" + }, + { + "algorithm": "sha256", + "hash": "0e06ab8858a76e1219e68c7573dfeba9dd1c0219476c5a44d5333b01d7e1743a", + "url": "https://files.pythonhosted.org/packages/92/28/631872d7fbc45527037060db8c838b47a129a6c09d2297d6dddcfa283cf2/watchdog-3.0.0-py3-none-manylinux2014_aarch64.whl" + }, + { + "algorithm": "sha256", + "hash": "3aa7f6a12e831ddfe78cdd4f8996af9cf334fd6346531b16cec61c3b3c0d8da0", + "url": "https://files.pythonhosted.org/packages/94/ce/70c65a6c4b0330129c402624d42f67ce82d6a0ba2036de67628aeffda3c1/watchdog-3.0.0-cp39-cp39-macosx_10_9_x86_64.whl" + }, + { + "algorithm": "sha256", + "hash": "4d98a320595da7a7c5a18fc48cb633c2e73cda78f93cac2ef42d42bf609a33f9", + "url": "https://files.pythonhosted.org/packages/95/a6/d6ef450393dac5734c63c40a131f66808d2e6f59f6165ab38c98fbe4e6ec/watchdog-3.0.0.tar.gz" + }, + { + "algorithm": "sha256", + "hash": "d00e6be486affb5781468457b21a6cbe848c33ef43f9ea4a73b4882e5f188a44", + "url": "https://files.pythonhosted.org/packages/c0/a2/4e3230bdc1fb878b152a2c66aa941732776f4545bd68135d490591d66713/watchdog-3.0.0-py3-none-manylinux2014_armv7l.whl" + }, + { + "algorithm": "sha256", + "hash": "4f94069eb16657d2c6faada4624c39464f65c05606af50bb7902e036e3219be3", + "url": "https://files.pythonhosted.org/packages/dc/89/3a3ce6dd01807ff918aec3bbcabc92ed1a7edc5bb2266c720bb39fec1bec/watchdog-3.0.0-cp38-cp38-macosx_11_0_arm64.whl" + }, + { + "algorithm": "sha256", + "hash": "8f3ceecd20d71067c7fd4c9e832d4e22584318983cabc013dbf3f70ea95de346", + "url": "https://files.pythonhosted.org/packages/ea/76/bef1c6f6ac18041234a9f3e8bc995d611e255c44f10433bfaf255968c269/watchdog-3.0.0-pp38-pypy38_pp73-macosx_10_9_x86_64.whl" + } + ], + "project_name": "watchdog", + "requires_dists": [ + "PyYAML>=3.10; extra == \"watchmedo\"" + ], + "requires_python": ">=3.7", + "version": "3.0.0" + }, { "artifacts": [ { @@ -4881,7 +4934,6 @@ "jsonschema<4,>=3", "kombu", "lockfile", - "logshipper", "mail-parser==3.15.0", "mock", "mongoengine", @@ -4896,7 +4948,6 @@ "prettytable", "prompt-toolkit<2", "psutil", - "pyinotify<=0.10,>=0.9.5; platform_system == \"Linux\"", "pymongo", "pyrabbit", "pysocks", @@ -4926,6 +4977,7 @@ "ujson", "unittest2", "virtualenv", + "watchdog", "webob", "webtest", "wheel", From ed3234846ba20e42f0f518dfa930586a33438901 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Mon, 29 Jan 2024 21:31:32 -0600 Subject: [PATCH 33/35] linux pack: Drop out-of-date README notes about pack config Pack config was removed for the linux pack in #3361 and #3475. Remove the README note as it no longer applies. --- contrib/linux/README.md | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/contrib/linux/README.md b/contrib/linux/README.md index 6c5100e2e8..6c708c810b 100644 --- a/contrib/linux/README.md +++ b/contrib/linux/README.md @@ -2,22 +2,6 @@ This pack contains actions for commonly used Linux commands and tools. -## Configuration - -* ``file_watch_sensor.file_paths`` - A list of paths to the files to monitor. - Note: Those need to be full paths to the files (e.g. ``/var/log/auth.log``) - and not directories (files don't need to exist yet when the sensor is ran - though). - -Example: - -```yaml ---- -file_watch_sensor: - file_paths: - - /opt/data/absolute_path_to_file.log -``` - ## Sensors ### FileWatchSensor From 465a8f41c642df40c444a1051922745c25e8e4d8 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Mon, 29 Jan 2024 21:40:56 -0600 Subject: [PATCH 34/35] update the linux pack README to explain how to use FileWatchSensor --- contrib/linux/README.md | 34 +++++++++++++++++++++++++++++---- contrib/linux/sensors/README.md | 2 +- 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/contrib/linux/README.md b/contrib/linux/README.md index 6c708c810b..b296ad9741 100644 --- a/contrib/linux/README.md +++ b/contrib/linux/README.md @@ -6,12 +6,38 @@ This pack contains actions for commonly used Linux commands and tools. ### FileWatchSensor -This sensor monitors specified files for new new lines. Once a new line is -detected, a trigger is emitted. +This sensor monitors files specified in rules (under trigger parameters) for new lines. +Once a new line is detected, a trigger instance is emitted. -### linux.file_watch.line trigger +#### Adding a file path to the file watch sensor -Example trigger payload: +To tell the FileWatchSensor to start watching a new file, define a rule that +- uses the `linux.file_watch.line` trigger type +- pass the `file_path` to watch under trigger parameters. + +For example, this rule would cause the sensor to start watching `/tmp/st2_test`: + +```yaml +--- +name: sample_rule_file_watch +description: Run echo on changes to /tmp/st2_test +enabled: false + +trigger: + type: linux.file_watch.line + parameters: + file_path: /tmp/st2_test + +action: + ref: core.local + parameters: + cmd: echo "{{trigger}}" +``` + + +#### linux.file_watch.line trigger + +Example trigger instance payload: ```json { diff --git a/contrib/linux/sensors/README.md b/contrib/linux/sensors/README.md index 084fcad6a6..743a0a3246 100644 --- a/contrib/linux/sensors/README.md +++ b/contrib/linux/sensors/README.md @@ -1,6 +1,6 @@ ## NOTICE -File watch sensor has been updated to use trigger with parameters supplied via a rule approach. Tailing a file path supplied via a config file is now deprecated. +File watch sensor has been updated to use trigger with parameters supplied via a rule approach. An example rule to supply a file path is as follows: From 36bacae26ec71610439511c5b74d8cf3ebe6b527 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Mon, 29 Jan 2024 22:20:06 -0600 Subject: [PATCH 35/35] linux pack: add LinuxFileWatchSensor.update_trigger --- contrib/linux/sensors/file_watch_sensor.py | 42 +++++++++++++++++++--- 1 file changed, 37 insertions(+), 5 deletions(-) diff --git a/contrib/linux/sensors/file_watch_sensor.py b/contrib/linux/sensors/file_watch_sensor.py index 10b8623fd3..77a702ff1c 100644 --- a/contrib/linux/sensors/file_watch_sensor.py +++ b/contrib/linux/sensors/file_watch_sensor.py @@ -427,20 +427,52 @@ def add_trigger(self, trigger): self.log.error('Received trigger type without "file_path" field.') return - trigger = trigger.get("ref", None) + trigger_ref = trigger.get("ref", None) - if not trigger: - raise Exception(f"Trigger {trigger} did not contain a ref.") + if not trigger_ref: + raise Exception(f"Trigger {trigger_ref} did not contain a ref.") self.tail_manager.tail_file(file_path, self._handle_line) self.file_ref[file_path] = trigger - self.log.info(f"Added file '{file_path}' ({trigger}) to watch list.") + self.log.info(f"Added file '{file_path}' ({trigger_ref}) to watch list.") self.tail_manager.start() def update_trigger(self, trigger): - pass + file_path = trigger["parameters"].get("file_path", None) + + if not file_path: + self.log.error('Received trigger type without "file_path" field.') + return + + trigger_ref = trigger.get("ref", None) + + if file_path in self.file_ref: + self.log.debug( + f"No update required as file '{file_path}' ({trigger_ref}) already in watch list." + ) + return + + if not trigger_ref: + raise Exception(f"Trigger {trigger_ref} did not contain a ref.") + + for old_file_path, ref in self.file_ref.items(): + if ref == trigger_ref: + self.tail_manager.stop_tailing_file(old_file_path, self._handle_line) + self.file_ref.pop(old_file_path) + + self.tail_manager.tail_file(file_path, self._handle_line) + self.file_ref[file_path] = trigger + + self.log.info( + f"Updated to add file '{file_path}' instead of '{old_file_path}' ({trigger_ref}) in watch list." + ) + break + + if file_path not in self.file_ref: + # Maybe the add_trigger message was missed. + self.add_trigger(trigger) def remove_trigger(self, trigger): file_path = trigger["parameters"].get("file_path", None)