diff options
Diffstat (limited to 'powerline/lib')
| -rw-r--r-- | powerline/lib/__init__.py | 28 | ||||
| -rw-r--r-- | powerline/lib/config.py | 218 | ||||
| -rwxr-xr-x | powerline/lib/debug.py | 97 | ||||
| -rw-r--r-- | powerline/lib/dict.py | 88 | ||||
| -rw-r--r-- | powerline/lib/encoding.py | 125 | ||||
| -rw-r--r-- | powerline/lib/humanize_bytes.py | 25 | ||||
| -rw-r--r-- | powerline/lib/inotify.py | 184 | ||||
| -rw-r--r-- | powerline/lib/memoize.py | 42 | ||||
| -rw-r--r-- | powerline/lib/monotonic.py | 100 | ||||
| -rw-r--r-- | powerline/lib/overrides.py | 80 | ||||
| -rw-r--r-- | powerline/lib/path.py | 18 | ||||
| -rw-r--r-- | powerline/lib/shell.py | 133 | ||||
| -rw-r--r-- | powerline/lib/threaded.py | 262 | ||||
| -rw-r--r-- | powerline/lib/unicode.py | 283 | ||||
| -rw-r--r-- | powerline/lib/url.py | 17 | ||||
| -rw-r--r-- | powerline/lib/vcs/__init__.py | 276 | ||||
| -rw-r--r-- | powerline/lib/vcs/bzr.py | 108 | ||||
| -rw-r--r-- | powerline/lib/vcs/git.py | 208 | ||||
| -rw-r--r-- | powerline/lib/vcs/mercurial.py | 88 | ||||
| -rw-r--r-- | powerline/lib/watcher/__init__.py | 76 | ||||
| -rw-r--r-- | powerline/lib/watcher/inotify.py | 268 | ||||
| -rw-r--r-- | powerline/lib/watcher/stat.py | 44 | ||||
| -rw-r--r-- | powerline/lib/watcher/tree.py | 90 | ||||
| -rw-r--r-- | powerline/lib/watcher/uv.py | 207 |
24 files changed, 3065 insertions, 0 deletions
diff --git a/powerline/lib/__init__.py b/powerline/lib/__init__.py new file mode 100644 index 0000000..2a5fbd0 --- /dev/null +++ b/powerline/lib/__init__.py @@ -0,0 +1,28 @@ +# vim:fileencoding=utf-8:noet +from __future__ import (unicode_literals, division, absolute_import, print_function) + +from functools import wraps + + +def wraps_saveargs(wrapped): + def dec(wrapper): + r = wraps(wrapped)(wrapper) + r.powerline_origin = getattr(wrapped, 'powerline_origin', wrapped) + return r + return dec + + +def add_divider_highlight_group(highlight_group): + def dec(func): + @wraps_saveargs(func) + def f(**kwargs): + r = func(**kwargs) + if r: + return [{ + 'contents': r, + 'divider_highlight_group': highlight_group, + }] + else: + return None + return f + return dec diff --git a/powerline/lib/config.py b/powerline/lib/config.py new file mode 100644 index 0000000..0c95e47 --- /dev/null +++ b/powerline/lib/config.py @@ -0,0 +1,218 @@ +# vim:fileencoding=utf-8:noet +from __future__ import (unicode_literals, division, absolute_import, print_function) + +import json +import codecs + +from copy import deepcopy +from threading import Event, Lock +from collections import defaultdict + +from powerline.lib.threaded import MultiRunnedThread +from powerline.lib.watcher import create_file_watcher + + +def open_file(path): + return codecs.open(path, encoding='utf-8') + + +def load_json_config(config_file_path, load=json.load, open_file=open_file): + with open_file(config_file_path) as config_file_fp: + return load(config_file_fp) + + +class DummyWatcher(object): + def __call__(self, *args, **kwargs): + return False + + def watch(self, *args, **kwargs): + pass + + +class DeferredWatcher(object): + def __init__(self, *args, **kwargs): + self.args = args + self.kwargs = kwargs + self.calls = [] + + def __call__(self, *args, **kwargs): + self.calls.append(('__call__', args, kwargs)) + + def watch(self, *args, **kwargs): + self.calls.append(('watch', args, kwargs)) + + def unwatch(self, *args, **kwargs): + self.calls.append(('unwatch', args, kwargs)) + + def transfer_calls(self, watcher): + for attr, args, kwargs in self.calls: + getattr(watcher, attr)(*args, **kwargs) + + +class ConfigLoader(MultiRunnedThread): + def __init__(self, shutdown_event=None, watcher=None, watcher_type=None, load=load_json_config, run_once=False): + super(ConfigLoader, self).__init__() + self.shutdown_event = shutdown_event or Event() + if run_once: + self.watcher = DummyWatcher() + self.watcher_type = 'dummy' + else: + self.watcher = watcher or DeferredWatcher() + if watcher: + if not watcher_type: + raise ValueError('When specifying watcher you must also specify watcher type') + self.watcher_type = watcher_type + else: + self.watcher_type = 'deferred' + self._load = load + + self.pl = None + self.interval = None + + self.lock = Lock() + + self.watched = defaultdict(set) + self.missing = defaultdict(set) + self.loaded = {} + + def set_watcher(self, watcher_type, force=False): + if watcher_type == self.watcher_type: + return + watcher = create_file_watcher(self.pl, watcher_type) + with self.lock: + if self.watcher_type == 'deferred': + self.watcher.transfer_calls(watcher) + self.watcher = watcher + self.watcher_type = watcher_type + + def set_pl(self, pl): + self.pl = pl + + def set_interval(self, interval): + self.interval = interval + + def register(self, function, path): + '''Register function that will be run when file changes. + + :param function function: + Function that will be called when file at the given path changes. + :param str path: + Path that will be watched for. + ''' + with self.lock: + self.watched[path].add(function) + self.watcher.watch(path) + + def register_missing(self, condition_function, function, key): + '''Register any function that will be called with given key each + interval seconds (interval is defined at __init__). Its result is then + passed to ``function``, but only if the result is true. + + :param function condition_function: + Function which will be called each ``interval`` seconds. All + exceptions from it will be logged and ignored. IOError exception + will be ignored without logging. + :param function function: + Function which will be called if condition_function returns + something that is true. Accepts result of condition_function as an + argument. + :param str key: + Any value, it will be passed to condition_function on each call. + + Note: registered functions will be automatically removed if + condition_function results in something true. + ''' + with self.lock: + self.missing[key].add((condition_function, function)) + + def unregister_functions(self, removed_functions): + '''Unregister files handled by these functions. + + :param set removed_functions: + Set of functions previously passed to ``.register()`` method. + ''' + with self.lock: + for path, functions in list(self.watched.items()): + functions -= removed_functions + if not functions: + self.watched.pop(path) + self.loaded.pop(path, None) + + def unregister_missing(self, removed_functions): + '''Unregister files handled by these functions. + + :param set removed_functions: + Set of pairs (2-tuples) representing ``(condition_function, + function)`` function pairs previously passed as an arguments to + ``.register_missing()`` method. + ''' + with self.lock: + for key, functions in list(self.missing.items()): + functions -= removed_functions + if not functions: + self.missing.pop(key) + + def load(self, path): + try: + # No locks: GIL does what we need + return deepcopy(self.loaded[path]) + except KeyError: + r = self._load(path) + self.loaded[path] = deepcopy(r) + return r + + def update(self): + toload = [] + with self.lock: + for path, functions in self.watched.items(): + for function in functions: + try: + modified = self.watcher(path) + except OSError as e: + modified = True + self.exception('Error while running watcher for path {0}: {1}', path, str(e)) + else: + if modified: + toload.append(path) + if modified: + function(path) + with self.lock: + for key, functions in list(self.missing.items()): + for condition_function, function in list(functions): + try: + path = condition_function(key) + except IOError: + pass + except Exception as e: + self.exception('Error while running condition function for key {0}: {1}', key, str(e)) + else: + if path: + toload.append(path) + function(path) + functions.remove((condition_function, function)) + if not functions: + self.missing.pop(key) + for path in toload: + try: + self.loaded[path] = deepcopy(self._load(path)) + except Exception as e: + self.exception('Error while loading {0}: {1}', path, str(e)) + try: + self.loaded.pop(path) + except KeyError: + pass + try: + self.loaded.pop(path) + except KeyError: + pass + + def run(self): + while self.interval is not None and not self.shutdown_event.is_set(): + self.update() + self.shutdown_event.wait(self.interval) + + def exception(self, msg, *args, **kwargs): + if self.pl: + self.pl.exception(msg, prefix='config_loader', *args, **kwargs) + else: + raise diff --git a/powerline/lib/debug.py b/powerline/lib/debug.py new file mode 100755 index 0000000..515e8c4 --- /dev/null +++ b/powerline/lib/debug.py @@ -0,0 +1,97 @@ +# vim:fileencoding=utf-8:noet +from __future__ import (unicode_literals, division, absolute_import, print_function) + +import gc +import sys + +from types import FrameType +from itertools import chain + + +# From http://code.activestate.com/recipes/523004-find-cyclical-references/ +def print_cycles(objects, outstream=sys.stdout, show_progress=False): + '''Find reference cycles + + :param list objects: + A list of objects to find cycles in. It is often useful to pass in + gc.garbage to find the cycles that are preventing some objects from + being garbage collected. + :param file outstream: + The stream for output. + :param bool show_progress: + If True, print the number of objects reached as they are found. + ''' + def print_path(path): + for i, step in enumerate(path): + # next “wraps around” + next = path[(i + 1) % len(path)] + + outstream.write(' %s -- ' % str(type(step))) + written = False + if isinstance(step, dict): + for key, val in step.items(): + if val is next: + outstream.write('[%s]' % repr(key)) + written = True + break + if key is next: + outstream.write('[key] = %s' % repr(val)) + written = True + break + elif isinstance(step, (list, tuple)): + for i, item in enumerate(step): + if item is next: + outstream.write('[%d]' % i) + written = True + elif getattr(type(step), '__getattribute__', None) in (object.__getattribute__, type.__getattribute__): + for attr in chain(dir(step), getattr(step, '__dict__', ())): + if getattr(step, attr, None) is next: + try: + outstream.write('%r.%s' % (step, attr)) + except TypeError: + outstream.write('.%s' % (step, attr)) + written = True + break + if not written: + outstream.write(repr(step)) + outstream.write(' ->\n') + outstream.write('\n') + + def recurse(obj, start, all, current_path): + if show_progress: + outstream.write('%d\r' % len(all)) + + all[id(obj)] = None + + referents = gc.get_referents(obj) + for referent in referents: + # If we’ve found our way back to the start, this is + # a cycle, so print it out + if referent is start: + try: + outstream.write('Cyclic reference: %r\n' % referent) + except TypeError: + try: + outstream.write('Cyclic reference: %i (%r)\n' % (id(referent), type(referent))) + except TypeError: + outstream.write('Cyclic reference: %i\n' % id(referent)) + print_path(current_path) + + # Don’t go back through the original list of objects, or + # through temporary references to the object, since those + # are just an artifact of the cycle detector itself. + elif referent is objects or isinstance(referent, FrameType): + continue + + # We haven’t seen this object before, so recurse + elif id(referent) not in all: + recurse(referent, start, all, current_path + (obj,)) + + for obj in objects: + # We are not interested in non-powerline cyclic references + try: + if not type(obj).__module__.startswith('powerline'): + continue + except AttributeError: + continue + recurse(obj, obj, {}, ()) diff --git a/powerline/lib/dict.py b/powerline/lib/dict.py new file mode 100644 index 0000000..c06ab30 --- /dev/null +++ b/powerline/lib/dict.py @@ -0,0 +1,88 @@ +# vim:fileencoding=utf-8:noet +from __future__ import (unicode_literals, division, absolute_import, print_function) + + +REMOVE_THIS_KEY = object() + + +def mergeargs(argvalue, remove=False): + if not argvalue: + return None + r = {} + for subval in argvalue: + mergedicts(r, dict([subval]), remove=remove) + return r + + +def _clear_special_values(d): + '''Remove REMOVE_THIS_KEY values from dictionary + ''' + l = [d] + while l: + i = l.pop() + pops = [] + for k, v in i.items(): + if v is REMOVE_THIS_KEY: + pops.append(k) + elif isinstance(v, dict): + l.append(v) + for k in pops: + i.pop(k) + + +def mergedicts(d1, d2, remove=True): + '''Recursively merge two dictionaries + + First dictionary is modified in-place. + ''' + _setmerged(d1, d2) + for k in d2: + if k in d1 and isinstance(d1[k], dict) and isinstance(d2[k], dict): + mergedicts(d1[k], d2[k], remove) + elif remove and d2[k] is REMOVE_THIS_KEY: + d1.pop(k, None) + else: + if remove and isinstance(d2[k], dict): + _clear_special_values(d2[k]) + d1[k] = d2[k] + + +def mergedefaults(d1, d2): + '''Recursively merge two dictionaries, keeping existing values + + First dictionary is modified in-place. + ''' + for k in d2: + if k in d1 and isinstance(d1[k], dict) and isinstance(d2[k], dict): + mergedefaults(d1[k], d2[k]) + else: + d1.setdefault(k, d2[k]) + + +def _setmerged(d1, d2): + if hasattr(d1, 'setmerged'): + d1.setmerged(d2) + + +def mergedicts_copy(d1, d2): + '''Recursively merge two dictionaries. + + Dictionaries are not modified. Copying happens only if necessary. Assumes + that first dictionary supports .copy() method. + ''' + ret = d1.copy() + _setmerged(ret, d2) + for k in d2: + if k in d1 and isinstance(d1[k], dict) and isinstance(d2[k], dict): + ret[k] = mergedicts_copy(d1[k], d2[k]) + else: + ret[k] = d2[k] + return ret + + +def updated(d, *args, **kwargs): + '''Copy dictionary and update it with provided arguments + ''' + d = d.copy() + d.update(*args, **kwargs) + return d diff --git a/powerline/lib/encoding.py b/powerline/lib/encoding.py new file mode 100644 index 0000000..76a51d8 --- /dev/null +++ b/powerline/lib/encoding.py @@ -0,0 +1,125 @@ +# vim:fileencoding=utf-8:noet + +'''Encodings support + +This is the only module from which functions obtaining encoding should be +exported. Note: you should always care about errors= argument since it is not +guaranteed that encoding returned by some function can encode/decode given +string. + +All functions in this module must always return a valid encoding. Most of them +are not thread-safe. +''' + +from __future__ import (unicode_literals, division, absolute_import, print_function) + +import sys +import locale + + +def get_preferred_file_name_encoding(): + '''Get preferred file name encoding + ''' + return ( + sys.getfilesystemencoding() + or locale.getpreferredencoding() + or 'utf-8' + ) + + +def get_preferred_file_contents_encoding(): + '''Get encoding preferred for file contents + ''' + return ( + locale.getpreferredencoding() + or 'utf-8' + ) + + +def get_preferred_output_encoding(): + '''Get encoding that should be used for printing strings + + .. warning:: + Falls back to ASCII, so that output is most likely to be displayed + correctly. + ''' + if hasattr(locale, 'LC_MESSAGES'): + return ( + locale.getlocale(locale.LC_MESSAGES)[1] + or locale.getdefaultlocale()[1] + or 'ascii' + ) + + return ( + locale.getdefaultlocale()[1] + or 'ascii' + ) + + +def get_preferred_input_encoding(): + '''Get encoding that should be used for reading shell command output + + .. warning:: + Falls back to latin1 so that function is less likely to throw as decoded + output is primary searched for ASCII values. + ''' + if hasattr(locale, 'LC_MESSAGES'): + return ( + locale.getlocale(locale.LC_MESSAGES)[1] + or locale.getdefaultlocale()[1] + or 'latin1' + ) + + return ( + locale.getdefaultlocale()[1] + or 'latin1' + ) + + +def get_preferred_arguments_encoding(): + '''Get encoding that should be used for command-line arguments + + .. warning:: + Falls back to latin1 so that function is less likely to throw as + non-ASCII command-line arguments most likely contain non-ASCII + filenames and screwing them up due to unidentified locale is not much of + a problem. + ''' + return ( + locale.getdefaultlocale()[1] + or 'latin1' + ) + + +def get_preferred_environment_encoding(): + '''Get encoding that should be used for decoding environment variables + ''' + return ( + locale.getpreferredencoding() + or 'utf-8' + ) + + +def get_unicode_writer(stream=sys.stdout, encoding=None, errors='replace'): + '''Get function which will write unicode string to the given stream + + Writing is done using encoding returned by + :py:func:`get_preferred_output_encoding`. + + :param file stream: + Stream to write to. Default value is :py:attr:`sys.stdout`. + :param str encoding: + Determines which encoding to use. If this argument is specified then + :py:func:`get_preferred_output_encoding` is not used. + :param str errors: + Determines what to do with characters which cannot be encoded. See + ``errors`` argument of :py:func:`codecs.encode`. + + :return: Callable which writes unicode string to the given stream using + the preferred output encoding. + ''' + encoding = encoding or get_preferred_output_encoding() + if sys.version_info < (3,) or not hasattr(stream, 'buffer'): + return lambda s: stream.write(s.encode(encoding, errors)) + else: + return lambda s: stream.buffer.write(s.encode(encoding, errors)) diff --git a/powerline/lib/humanize_bytes.py b/powerline/lib/humanize_bytes.py new file mode 100644 index 0000000..c98a117 --- /dev/null +++ b/powerline/lib/humanize_bytes.py @@ -0,0 +1,25 @@ +# vim:fileencoding=utf-8:noet +from __future__ import (unicode_literals, division, absolute_import, print_function) + +from math import log + + +unit_list = tuple(zip(['', 'k', 'M', 'G', 'T', 'P'], [0, 0, 1, 2, 2, 2])) + + +def humanize_bytes(num, suffix='B', si_prefix=False): + '''Return a human friendly byte representation. + + Modified version from http://stackoverflow.com/questions/1094841 + ''' + if num == 0: + return '0 ' + suffix + div = 1000 if si_prefix else 1024 + exponent = min(int(log(num, div)) if num else 0, len(unit_list) - 1) + quotient = float(num) / div ** exponent + unit, decimals = unit_list[exponent] + if unit and not si_prefix: + unit = unit.upper() + 'i' + return ('{{quotient:.{decimals}f}} {{unit}}{{suffix}}' + .format(decimals=decimals) + .format(quotient=quotient, unit=unit, suffix=suffix)) diff --git a/powerline/lib/inotify.py b/powerline/lib/inotify.py new file mode 100644 index 0000000..8b74a7f --- /dev/null +++ b/powerline/lib/inotify.py @@ -0,0 +1,184 @@ +# vim:fileencoding=utf-8:noet +from __future__ import (unicode_literals, division, absolute_import, print_function) + +import sys +import os +import errno +import ctypes +import struct + +from ctypes.util import find_library + +from powerline.lib.encoding import get_preferred_file_name_encoding + + +__copyright__ = '2013, Kovid Goyal <kovid at kovidgoyal.net>' +__docformat__ = 'restructuredtext en' + + +class INotifyError(Exception): + pass + + +_inotify = None + + +def load_inotify(): + ''' Initialize the inotify library ''' + global _inotify + if _inotify is None: + if hasattr(sys, 'getwindowsversion'): + # On windows abort before loading the C library. Windows has + # multiple, incompatible C runtimes, and we have no way of knowing + # if the one chosen by ctypes is compatible with the currently + # loaded one. + raise INotifyError('INotify not available on windows') + if sys.platform == 'darwin': + raise INotifyError('INotify not available on OS X') + if not hasattr(ctypes, 'c_ssize_t'): + raise INotifyError('You need python >= 2.7 to use inotify') + name = find_library('c') + if not name: + raise INotifyError('Cannot find C library') + libc = ctypes.CDLL(name, use_errno=True) + for function in ('inotify_add_watch', 'inotify_init1', 'inotify_rm_watch'): + if not hasattr(libc, function): + raise INotifyError('libc is too old') + # inotify_init1() + prototype = ctypes.CFUNCTYPE(ctypes.c_int, ctypes.c_int, use_errno=True) + init1 = prototype(('inotify_init1', libc), ((1, 'flags', 0),)) + + # inotify_add_watch() + prototype = ctypes.CFUNCTYPE(ctypes.c_int, ctypes.c_int, ctypes.c_char_p, ctypes.c_uint32, use_errno=True) + add_watch = prototype(('inotify_add_watch', libc), ( + (1, 'fd'), (1, 'pathname'), (1, 'mask'))) + + # inotify_rm_watch() + prototype = ctypes.CFUNCTYPE(ctypes.c_int, ctypes.c_int, ctypes.c_int, use_errno=True) + rm_watch = prototype(('inotify_rm_watch', libc), ( + (1, 'fd'), (1, 'wd'))) + + # read() + prototype = ctypes.CFUNCTYPE(ctypes.c_ssize_t, ctypes.c_int, ctypes.c_void_p, ctypes.c_size_t, use_errno=True) + read = prototype(('read', libc), ( + (1, 'fd'), (1, 'buf'), (1, 'count'))) + _inotify = (init1, add_watch, rm_watch, read) + return _inotify + + +class INotify(object): + + # See <sys/inotify.h> for the flags defined below + + # Supported events suitable for MASK parameter of INOTIFY_ADD_WATCH. + ACCESS = 0x00000001 # File was accessed. + MODIFY = 0x00000002 # File was modified. + ATTRIB = 0x00000004 # Metadata changed. + CLOSE_WRITE = 0x00000008 # Writtable file was closed. + CLOSE_NOWRITE = 0x00000010 # Unwrittable file closed. + OPEN = 0x00000020 # File was opened. + MOVED_FROM = 0x00000040 # File was moved from X. + MOVED_TO = 0x00000080 # File was moved to Y. + CREATE = 0x00000100 # Subfile was created. + DELETE = 0x00000200 # Subfile was deleted. + DELETE_SELF = 0x00000400 # Self was deleted. + MOVE_SELF = 0x00000800 # Self was moved. + + # Events sent by the kernel. + UNMOUNT = 0x00002000 # Backing fs was unmounted. + Q_OVERFLOW = 0x00004000 # Event queued overflowed. + IGNORED = 0x00008000 # File was ignored. + + # Helper events. + CLOSE = (CLOSE_WRITE | CLOSE_NOWRITE) # Close. + MOVE = (MOVED_FROM | MOVED_TO) # Moves. + + # Special flags. + ONLYDIR = 0x01000000 # Only watch the path if it is a directory. + DONT_FOLLOW = 0x02000000 # Do not follow a sym link. + EXCL_UNLINK = 0x04000000 # Exclude events on unlinked objects. + MASK_ADD = 0x20000000 # Add to the mask of an already existing watch. + ISDIR = 0x40000000 # Event occurred against dir. + ONESHOT = 0x80000000 # Only send event once. + + # All events which a program can wait on. + ALL_EVENTS = ( + ACCESS | MODIFY | ATTRIB | CLOSE_WRITE | CLOSE_NOWRITE | OPEN | + MOVED_FROM | MOVED_TO | CREATE | DELETE | DELETE_SELF | MOVE_SELF + ) + + # See <bits/inotify.h> + CLOEXEC = 0x80000 + NONBLOCK = 0x800 + + def __init__(self, cloexec=True, nonblock=True): + self._init1, self._add_watch, self._rm_watch, self._read = load_inotify() + flags = 0 + if cloexec: + flags |= self.CLOEXEC + if nonblock: + flags |= self.NONBLOCK + self._inotify_fd = self._init1(flags) + if self._inotify_fd == -1: + raise INotifyError(os.strerror(ctypes.get_errno())) + + self._buf = ctypes.create_string_buffer(5000) + self.fenc = get_preferred_file_name_encoding() + self.hdr = struct.Struct(b'iIII') + # We keep a reference to os to prevent it from being deleted + # during interpreter shutdown, which would lead to errors in the + # __del__ method + self.os = os + + def handle_error(self): + eno = ctypes.get_errno() + extra = '' + if eno == errno.ENOSPC: + extra = 'You may need to increase the inotify limits on your system, via /proc/sys/fs/inotify/max_user_*' + raise OSError(eno, self.os.strerror(eno) + str(extra)) + + def __del__(self): + # This method can be called during interpreter shutdown, which means we + # must do the absolute minimum here. Note that there could be running + # daemon threads that are trying to call other methods on this object. + try: + self.os.close(self._inotify_fd) + except (AttributeError, TypeError): + pass + + def close(self): + if hasattr(self, '_inotify_fd'): + self.os.close(self._inotify_fd) + del self.os + del self._add_watch + del self._rm_watch + del self._inotify_fd + + def read(self, get_name=True): + buf = [] + while True: + num = self._read(self._inotify_fd, self._buf, len(self._buf)) + if num == 0: + break + if num < 0: + en = ctypes.get_errno() + if en == errno.EAGAIN: + break # No more data + if en == errno.EINTR: + continue # Interrupted, try again + raise OSError(en, self.os.strerror(en)) + buf.append(self._buf.raw[:num]) + raw = b''.join(buf) + pos = 0 + lraw = len(raw) + while lraw - pos >= self.hdr.size: + wd, mask, cookie, name_len = self.hdr.unpack_from(raw, pos) + pos += self.hdr.size + name = None + if get_name: + name = raw[pos:pos + name_len].rstrip(b'\0') + pos += name_len + self.process_event(wd, mask, cookie, name) + + def process_event(self, *args): + raise NotImplementedError() diff --git a/powerline/lib/memoize.py b/powerline/lib/memoize.py new file mode 100644 index 0000000..cedbe45 --- /dev/null +++ b/powerline/lib/memoize.py @@ -0,0 +1,42 @@ +# vim:fileencoding=utf-8:noet +from __future__ import (unicode_literals, division, absolute_import, print_function) + +from functools import wraps + +from powerline.lib.monotonic import monotonic + + +def default_cache_key(**kwargs): + return frozenset(kwargs.items()) + + +class memoize(object): + '''Memoization decorator with timeout.''' + def __init__(self, timeout, cache_key=default_cache_key, cache_reg_func=None): + self.timeout = timeout + self.cache_key = cache_key + self.cache = {} + self.cache_reg_func = cache_reg_func + + def __call__(self, func): + @wraps(func) + def decorated_function(**kwargs): + if self.cache_reg_func: + self.cache_reg_func(self.cache) + self.cache_reg_func = None + + key = self.cache_key(**kwargs) + try: + cached = self.cache.get(key, None) + except TypeError: + return func(**kwargs) + # Handle case when time() appears to be less then cached['time'] due + # to clock updates. Not applicable for monotonic clock, but this + # case is currently rare. + if cached is None or not (cached['time'] < monotonic() < cached['time'] + self.timeout): + cached = self.cache[key] = { + 'result': func(**kwargs), + 'time': monotonic(), + } + return cached['result'] + return decorated_function diff --git a/powerline/lib/monotonic.py b/powerline/lib/monotonic.py new file mode 100644 index 0000000..cd7c414 --- /dev/null +++ b/powerline/lib/monotonic.py @@ -0,0 +1,100 @@ +# vim:fileencoding=utf-8:noet +from __future__ import (unicode_literals, division, absolute_import, print_function) + +try: + try: + # >=python-3.3, Unix + from time import clock_gettime + try: + # >={kernel}-sources-2.6.28 + from time import CLOCK_MONOTONIC_RAW as CLOCK_ID + except ImportError: + from time import CLOCK_MONOTONIC as CLOCK_ID + + monotonic = lambda: clock_gettime(CLOCK_ID) + except ImportError: + # >=python-3.3 + from time import monotonic +except ImportError: + import ctypes + import sys + + try: + if sys.platform == 'win32': + # Windows only + GetTickCount64 = ctypes.windll.kernel32.GetTickCount64 + GetTickCount64.restype = ctypes.c_ulonglong + + def monotonic(): + return GetTickCount64() / 1000 + + elif sys.platform == 'darwin': + # Mac OS X + from ctypes.util import find_library + + libc_name = find_library('c') + if not libc_name: + raise OSError + + libc = ctypes.CDLL(libc_name, use_errno=True) + + mach_absolute_time = libc.mach_absolute_time + mach_absolute_time.argtypes = () + mach_absolute_time.restype = ctypes.c_uint64 + + class mach_timebase_info_data_t(ctypes.Structure): + _fields_ = ( + ('numer', ctypes.c_uint32), + ('denom', ctypes.c_uint32), + ) + mach_timebase_info_data_p = ctypes.POINTER(mach_timebase_info_data_t) + + _mach_timebase_info = libc.mach_timebase_info + _mach_timebase_info.argtypes = (mach_timebase_info_data_p,) + _mach_timebase_info.restype = ctypes.c_int + + def mach_timebase_info(): + timebase = mach_timebase_info_data_t() + _mach_timebase_info(ctypes.byref(timebase)) + return (timebase.numer, timebase.denom) + + timebase = mach_timebase_info() + factor = timebase[0] / timebase[1] * 1e-9 + + def monotonic(): + return mach_absolute_time() * factor + else: + # linux only (no librt on OS X) + import os + + # See <bits/time.h> + CLOCK_MONOTONIC = 1 + CLOCK_MONOTONIC_RAW = 4 + + class timespec(ctypes.Structure): + _fields_ = ( + ('tv_sec', ctypes.c_long), + ('tv_nsec', ctypes.c_long) + ) + tspec = timespec() + + librt = ctypes.CDLL('librt.so.1', use_errno=True) + clock_gettime = librt.clock_gettime + clock_gettime.argtypes = [ctypes.c_int, ctypes.POINTER(timespec)] + + if clock_gettime(CLOCK_MONOTONIC_RAW, ctypes.pointer(tspec)) == 0: + # >={kernel}-sources-2.6.28 + clock_id = CLOCK_MONOTONIC_RAW + elif clock_gettime(CLOCK_MONOTONIC, ctypes.pointer(tspec)) == 0: + clock_id = CLOCK_MONOTONIC + else: + raise OSError + + def monotonic(): + if clock_gettime(CLOCK_MONOTONIC, ctypes.pointer(tspec)) != 0: + errno_ = ctypes.get_errno() + raise OSError(errno_, os.strerror(errno_)) + return tspec.tv_sec + tspec.tv_nsec / 1e9 + + except: + from time import time as monotonic # NOQA diff --git a/powerline/lib/overrides.py b/powerline/lib/overrides.py new file mode 100644 index 0000000..3257d98 --- /dev/null +++ b/powerline/lib/overrides.py @@ -0,0 +1,80 @@ +# vim:fileencoding=utf-8:noet +from __future__ import (unicode_literals, division, absolute_import, print_function) + +import json + +from powerline.lib.dict import REMOVE_THIS_KEY + + +def parse_value(s): + '''Convert string to Python object + + Rules: + + * Empty string means that corresponding key should be removed from the + dictionary. + * Strings that start with a minus, digit or with some character that starts + JSON collection or string object are parsed as JSON. + * JSON special values ``null``, ``true``, ``false`` (case matters) are + parsed as JSON. + * All other values are considered to be raw strings. + + :param str s: Parsed string. + + :return: Python object. + ''' + if not s: + return REMOVE_THIS_KEY + elif s[0] in '"{[0123456789-' or s in ('null', 'true', 'false'): + return json.loads(s) + else: + return s + + +def keyvaluesplit(s): + '''Split K1.K2=VAL into K1.K2 and parsed VAL + ''' + if '=' not in s: + raise TypeError('Option must look like option=json_value') + if s[0] == '_': + raise ValueError('Option names must not start with `_\'') + idx = s.index('=') + o = s[:idx] + val = parse_value(s[idx + 1:]) + return (o, val) + + +def parsedotval(s): + '''Parse K1.K2=VAL into {"K1":{"K2":VAL}} + + ``VAL`` is processed according to rules defined in :py:func:`parse_value`. + ''' + if type(s) is tuple: + o, val = s + val = parse_value(val) + else: + o, val = keyvaluesplit(s) + + keys = o.split('.') + if len(keys) > 1: + r = (keys[0], {}) + rcur = r[1] + for key in keys[1:-1]: + rcur[key] = {} + rcur = rcur[key] + rcur[keys[-1]] = val + return r + else: + return (o, val) + + +def parse_override_var(s): + '''Parse a semicolon-separated list of strings into a sequence of values + + Emits the same items in sequence as :py:func:`parsedotval` does. + ''' + return ( + parsedotval(item) + for item in s.split(';') + if item + ) diff --git a/powerline/lib/path.py b/powerline/lib/path.py new file mode 100644 index 0000000..49ff433 --- /dev/null +++ b/powerline/lib/path.py @@ -0,0 +1,18 @@ +# vim:fileencoding=utf-8:noet +from __future__ import (unicode_literals, division, absolute_import, print_function) + +import os + + +def realpath(path): + return os.path.abspath(os.path.realpath(path)) + + +def join(*components): + if any((isinstance(p, bytes) for p in components)): + return os.path.join(*[ + p if isinstance(p, bytes) else p.encode('ascii') + for p in components + ]) + else: + return os.path.join(*components) diff --git a/powerline/lib/shell.py b/powerline/lib/shell.py new file mode 100644 index 0000000..2082e82 --- /dev/null +++ b/powerline/lib/shell.py @@ -0,0 +1,133 @@ +# vim:fileencoding=utf-8:noet +from __future__ import (unicode_literals, division, absolute_import, print_function) + +import sys +import os + +from subprocess import Popen, PIPE +from functools import partial + +from powerline.lib.encoding import get_preferred_input_encoding, get_preferred_output_encoding + + +if sys.platform.startswith('win32'): + # Prevent windows from launching consoles when calling commands + # http://msdn.microsoft.com/en-us/library/windows/desktop/ms684863(v=vs.85).aspx + Popen = partial(Popen, creationflags=0x08000000) + + +def run_cmd(pl, cmd, stdin=None, strip=True): + '''Run command and return its stdout, stripped + + If running command fails returns None and logs failure to ``pl`` argument. + + :param PowerlineLogger pl: + Logger used to log failures. + :param list cmd: + Command which will be run. + :param str stdin: + String passed to command. May be None. + :param bool strip: + True if the result should be stripped. + ''' + try: + p = Popen(cmd, shell=False, stdout=PIPE, stdin=PIPE) + except OSError as e: + pl.exception('Could not execute command ({0}): {1}', e, cmd) + return None + else: + stdout, err = p.communicate( + stdin if stdin is None else stdin.encode(get_preferred_output_encoding())) + stdout = stdout.decode(get_preferred_input_encoding()) + return stdout.strip() if strip else stdout + + +def asrun(pl, ascript): + '''Run the given AppleScript and return the standard output and error.''' + return run_cmd(pl, ['osascript', '-'], ascript) + + +def readlines(cmd, cwd): + '''Run command and read its output, line by line + + :param list cmd: + Command which will be run. + :param str cwd: + Working directory of the command which will be run. + ''' + p = Popen(cmd, shell=False, stdout=PIPE, stderr=PIPE, cwd=cwd) + encoding = get_preferred_input_encoding() + p.stderr.close() + with p.stdout: + for line in p.stdout: + yield line[:-1].decode(encoding) + + +try: + from shutil import which +except ImportError: + # shutil.which was added in python-3.3. Here is what was added: + # Lib/shutil.py, commit 5abe28a9c8fe701ba19b1db5190863384e96c798 + def which(cmd, mode=os.F_OK | os.X_OK, path=None): + '''Given a command, mode, and a PATH string, return the path which + conforms to the given mode on the PATH, or None if there is no such + file. + + ``mode`` defaults to os.F_OK | os.X_OK. ``path`` defaults to the result + of ``os.environ.get('PATH')``, or can be overridden with a custom search + path. + ''' + # Check that a given file can be accessed with the correct mode. + # Additionally check that `file` is not a directory, as on Windows + # directories pass the os.access check. + def _access_check(fn, mode): + return ( + os.path.exists(fn) + and os.access(fn, mode) + and not os.path.isdir(fn) + ) + + # If we’re given a path with a directory part, look it up directly rather + # than referring to PATH directories. This includes checking relative to the + # current directory, e.g. ./script + if os.path.dirname(cmd): + if _access_check(cmd, mode): + return cmd + return None + + if path is None: + path = os.environ.get('PATH', os.defpath) + if not path: + return None + path = path.split(os.pathsep) + + if sys.platform == 'win32': + # The current directory takes precedence on Windows. + if os.curdir not in path: + path.insert(0, os.curdir) + + # PATHEXT is necessary to check on Windows. + pathext = os.environ.get('PATHEXT', '').split(os.pathsep) + # See if the given file matches any of the expected path extensions. + # This will allow us to short circuit when given 'python.exe'. + # If it does match, only test that one, otherwise we have to try + # others. + if any(cmd.lower().endswith(ext.lower()) for ext in pathext): + files = [cmd] + else: + files = [cmd + ext for ext in pathext] + else: + # On other platforms you don’t have things like PATHEXT to tell you + # what file suffixes are executable, so just pass on cmd as-is. + files = [cmd] + + seen = set() + for dir in path: + normdir = os.path.normcase(dir) + if normdir not in seen: + seen.add(normdir) + for thefile in files: + name = os.path.join(dir, thefile) + if _access_check(name, mode): + return name + return None diff --git a/powerline/lib/threaded.py b/powerline/lib/threaded.py new file mode 100644 index 0000000..e5a6b3e --- /dev/null +++ b/powerline/lib/threaded.py @@ -0,0 +1,262 @@ +# vim:fileencoding=utf-8:noet +from __future__ import (unicode_literals, division, absolute_import, print_function) + +from threading import Thread, Lock, Event +from types import MethodType + +from powerline.lib.monotonic import monotonic +from powerline.segments import Segment + + +class MultiRunnedThread(object): + daemon = True + + def __init__(self): + self.thread = None + + def is_alive(self): + return self.thread and self.thread.is_alive() + + def start(self): + self.shutdown_event.clear() + self.thread = Thread(target=self.run) + self.thread.daemon = self.daemon + self.thread.start() + + def join(self, *args, **kwargs): + if self.thread: + return self.thread.join(*args, **kwargs) + return None + + +class ThreadedSegment(Segment, MultiRunnedThread): + min_sleep_time = 0.1 + update_first = True + interval = 1 + daemon = False + + argmethods = ('render', 'set_state') + + def __init__(self): + super(ThreadedSegment, self).__init__() + self.run_once = True + self.crashed = False + self.crashed_value = None + self.update_value = None + self.updated = False + + def __call__(self, pl, update_first=True, **kwargs): + if self.run_once: + self.pl = pl + self.set_state(**kwargs) + update_value = self.get_update_value(True) + elif not self.is_alive(): + # Without this we will not have to wait long until receiving bug “I + # opened vim, but branch information is only shown after I move + # cursor”. + # + # If running once .update() is called in __call__. + self.start() + update_value = self.get_update_value(self.do_update_first) + else: + update_value = self.get_update_value(not self.updated) + + if self.crashed: + return self.crashed_value + + return self.render(update_value, update_first=update_first, pl=pl, **kwargs) + + def set_update_value(self): + try: + self.update_value = self.update(self.update_value) + except Exception as e: + self.exception('Exception while updating: {0}', str(e)) + self.crashed = True + except KeyboardInterrupt: + self.warn('Caught keyboard interrupt while updating') + self.crashed = True + else: + self.crashed = False + self.updated = True + + def get_update_value(self, update=False): + if update: + self.set_update_value() + return self.update_value + + def run(self): + if self.do_update_first: + start_time = monotonic() + while True: + self.shutdown_event.wait(max(self.interval - (monotonic() - start_time), self.min_sleep_time)) + if self.shutdown_event.is_set(): + break + start_time = monotonic() + self.set_update_value() + else: + while not self.shutdown_event.is_set(): + start_time = monotonic() + self.set_update_value() + self.shutdown_event.wait(max(self.interval - (monotonic() - start_time), self.min_sleep_time)) + + def shutdown(self): + self.shutdown_event.set() + if self.daemon and self.is_alive(): + # Give the worker thread a chance to shutdown, but don’t block for + # too long + self.join(0.01) + + def set_interval(self, interval=None): + # Allowing “interval” keyword in configuration. + # Note: Here **kwargs is needed to support foreign data, in subclasses + # it can be seen in a number of places in order to support + # .set_interval(). + interval = interval or getattr(self, 'interval') + self.interval = interval + + def set_state(self, interval=None, update_first=True, shutdown_event=None, **kwargs): + self.set_interval(interval) + self.shutdown_event = shutdown_event or Event() + self.do_update_first = update_first and self.update_first + self.updated = self.updated or (not self.do_update_first) + + def startup(self, pl, **kwargs): + self.run_once = False + self.pl = pl + self.daemon = pl.use_daemon_threads + + self.set_state(**kwargs) + + if not self.is_alive(): + self.start() + + def critical(self, *args, **kwargs): + self.pl.critical(prefix=self.__class__.__name__, *args, **kwargs) + + def exception(self, *args, **kwargs): + self.pl.exception(prefix=self.__class__.__name__, *args, **kwargs) + + def info(self, *args, **kwargs): + self.pl.info(prefix=self.__class__.__name__, *args, **kwargs) + + def error(self, *args, **kwargs): + self.pl.error(prefix=self.__class__.__name__, *args, **kwargs) + + def warn(self, *args, **kwargs): + self.pl.warn(prefix=self.__class__.__name__, *args, **kwargs) + + def debug(self, *args, **kwargs): + self.pl.debug(prefix=self.__class__.__name__, *args, **kwargs) + + def argspecobjs(self): + for name in self.argmethods: + try: + yield name, getattr(self, name) + except AttributeError: + pass + + def additional_args(self): + return (('interval', self.interval),) + + _omitted_args = { + 'render': (0,), + 'set_state': ('shutdown_event',), + } + + def omitted_args(self, name, method): + ret = self._omitted_args.get(name, ()) + if isinstance(getattr(self, name, None), MethodType): + ret = tuple((i + 1 if isinstance(i, int) else i for i in ret)) + return ret + + +class KwThreadedSegment(ThreadedSegment): + update_first = True + + argmethods = ('render', 'set_state', 'key', 'render_one') + + def __init__(self): + super(KwThreadedSegment, self).__init__() + self.updated = True + self.update_value = ({}, set()) + self.write_lock = Lock() + self.new_queries = [] + + @staticmethod + def key(**kwargs): + return frozenset(kwargs.items()) + + def render(self, update_value, update_first, key=None, after_update=False, **kwargs): + queries, crashed = update_value + if key is None: + key = self.key(**kwargs) + if key in crashed: + return self.crashed_value + + try: + update_state = queries[key][1] + except KeyError: + with self.write_lock: + self.new_queries.append(key) + if self.do_update_first or self.run_once: + if after_update: + self.error('internal error: value was not computed even though update_first was set') + update_state = None + else: + return self.render( + update_value=self.get_update_value(True), + update_first=False, + key=key, + after_update=True, + **kwargs + ) + else: + update_state = None + + return self.render_one(update_state, **kwargs) + + def update_one(self, crashed, updates, key): + try: + updates[key] = (monotonic(), self.compute_state(key)) + except Exception as e: + self.exception('Exception while computing state for {0!r}: {1}', key, str(e)) + crashed.add(key) + except KeyboardInterrupt: + self.warn('Interrupt while computing state for {0!r}', key) + crashed.add(key) + + def update(self, old_update_value): + updates = {} + crashed = set() + update_value = (updates, crashed) + queries = old_update_value[0] + + new_queries = self.new_queries + with self.write_lock: + self.new_queries = [] + + for key, (last_query_time, state) in queries.items(): + if last_query_time < monotonic() < last_query_time + self.interval: + updates[key] = (last_query_time, state) + else: + self.update_one(crashed, updates, key) + + for key in new_queries: + self.update_one(crashed, updates, key) + + return update_value + + def set_state(self, interval=None, update_first=True, shutdown_event=None, **kwargs): + self.set_interval(interval) + self.do_update_first = update_first and self.update_first + self.shutdown_event = shutdown_event or Event() + + @staticmethod + def render_one(update_state, **kwargs): + return update_state + + _omitted_args = { + 'render': ('update_value', 'key', 'after_update'), + 'set_state': ('shutdown_event',), + 'render_one': (0,), + } diff --git a/powerline/lib/unicode.py b/powerline/lib/unicode.py new file mode 100644 index 0000000..eeae387 --- /dev/null +++ b/powerline/lib/unicode.py @@ -0,0 +1,283 @@ +# vim:fileencoding=utf-8:noet +from __future__ import (unicode_literals, division, absolute_import, print_function) + +import sys +import codecs + +from unicodedata import east_asian_width, combining + +from powerline.lib.encoding import get_preferred_output_encoding + + +try: + from __builtin__ import unicode +except ImportError: + unicode = str + + +try: + from __builtin__ import unichr +except ImportError: + unichr = chr + + +if sys.maxunicode < 0x10FFFF: + _unichr = unichr + + def unichr(ch): + if ch <= sys.maxunicode: + return _unichr(ch) + else: + ch -= 0x10000 + return _unichr((ch >> 10) + 0xD800) + _unichr((ch & ((1 << 10) - 1)) + 0xDC00) + + +def u(s): + '''Return unicode instance assuming UTF-8 encoded string. + ''' + if type(s) is unicode: + return s + else: + return unicode(s, 'utf-8') + + +if sys.version_info < (3,): + def tointiter(s): + '''Convert a byte string to the sequence of integers + ''' + return (ord(c) for c in s) +else: + def tointiter(s): + '''Convert a byte string to the sequence of integers + ''' + return iter(s) + + +def powerline_decode_error(e): + if not isinstance(e, UnicodeDecodeError): + raise NotImplementedError + return (''.join(( + '<{0:02X}>'.format(c) + for c in tointiter(e.object[e.start:e.end]) + )), e.end) + + +codecs.register_error('powerline_decode_error', powerline_decode_error) + + +last_swe_idx = 0 + + +def register_strwidth_error(strwidth): + '''Create new encode errors handling method similar to ``replace`` + + Like ``replace`` this method uses question marks in place of the characters + that cannot be represented in the requested encoding. Unlike ``replace`` the + amount of question marks is identical to the amount of display cells + offending character occupies. Thus encoding ``…`` (U+2026, HORIZONTAL + ELLIPSIS) to ``latin1`` will emit one question mark, but encoding ``Ａ`` + (U+FF21, FULLWIDTH LATIN CAPITAL LETTER A) will emit two question marks. + + Since width of some characters depends on the terminal settings and + powerline knows how to respect them a single error handling method cannot be + used. Instead of it the generator function is used which takes ``strwidth`` + function (function that knows how to compute string width respecting all + needed settings) and emits new error handling method name. + + :param function strwidth: + Function that computs string width measured in display cells the string + occupies when displayed. + + :return: New error handling method name. + ''' + global last_swe_idx + last_swe_idx += 1 + + def powerline_encode_strwidth_error(e): + if not isinstance(e, UnicodeEncodeError): + raise NotImplementedError + return ('?' * strwidth(e.object[e.start:e.end]), e.end) + + ename = 'powerline_encode_strwidth_error_{0}'.format(last_swe_idx) + codecs.register_error(ename, powerline_encode_strwidth_error) + return ename + + +def out_u(s): + '''Return unicode string suitable for displaying + + Unlike other functions assumes get_preferred_output_encoding() first. Unlike + u() does not throw exceptions for invalid unicode strings. Unlike + safe_unicode() does throw an exception if object is not a string. + ''' + if isinstance(s, unicode): + return s + elif isinstance(s, bytes): + return unicode(s, get_preferred_output_encoding(), 'powerline_decode_error') + else: + raise TypeError('Expected unicode or bytes instance, got {0}'.format(repr(type(s)))) + + +def safe_unicode(s): + '''Return unicode instance without raising an exception. + + Order of assumptions: + * ASCII string or unicode object + * UTF-8 string + * Object with __str__() or __repr__() method that returns UTF-8 string or + unicode object (depending on python version) + * String in powerline.lib.encoding.get_preferred_output_encoding() encoding + * If everything failed use safe_unicode on last exception with which + everything failed + ''' + try: + try: + if type(s) is bytes: + return unicode(s, 'ascii') + else: + return unicode(s) + except UnicodeDecodeError: + try: + return unicode(s, 'utf-8') + except TypeError: + return unicode(str(s), 'utf-8') + except UnicodeDecodeError: + return unicode(s, get_preferred_output_encoding()) + except Exception as e: + return safe_unicode(e) + + +class FailedUnicode(unicode): + '''Builtin ``unicode`` subclass indicating fatal error + + If your code for some reason wants to determine whether `.render()` method + failed it should check returned string for being a FailedUnicode instance. + Alternatively you could subclass Powerline and override `.render()` method + to do what you like in place of catching the exception and returning + FailedUnicode. + ''' + pass + + +if sys.version_info < (3,): + def string(s): + if type(s) is not str: + return s.encode('utf-8') + else: + return s +else: + def string(s): + if type(s) is not str: + return s.decode('utf-8') + else: + return s + + +string.__doc__ = ( + '''Transform ``unicode`` or ``bytes`` object into ``str`` object + + On Python-2 this encodes ``unicode`` to ``bytes`` (which is ``str``) using + UTF-8 encoding; on Python-3 this decodes ``bytes`` to ``unicode`` (which is + ``str``) using UTF-8 encoding. + + Useful for functions that expect an ``str`` object in both unicode versions, + not caring about the semantic differences between them in Python-2 and + Python-3. + ''' +) + + +def surrogate_pair_to_character(high, low): + '''Transform a pair of surrogate codepoints to one codepoint + ''' + return 0x10000 + ((high - 0xD800) << 10) + (low - 0xDC00) + + +_strwidth_documentation = ( + '''Compute string width in display cells + + {0} + + :param dict width_data: + Dictionary which maps east_asian_width property values to strings + lengths. It is expected to contain the following keys and values (from + `East Asian Width annex <http://www.unicode.org/reports/tr11/>`_): + + === ====== =========================================================== + Key Value Description + === ====== =========================================================== + F 2 Fullwidth: all characters that are defined as Fullwidth in + the Unicode Standard [Unicode] by having a compatibility + decomposition of type <wide> to characters elsewhere in the + Unicode Standard that are implicitly narrow but unmarked. + H 1 Halfwidth: all characters that are explicitly defined as + Halfwidth in the Unicode Standard by having a compatibility + decomposition of type <narrow> to characters elsewhere in + the Unicode Standard that are implicitly wide but unmarked, + plus U+20A9 ₩ WON SIGN. + W 2 Wide: all other characters that are always wide. These + characters occur only in the context of East Asian + typography where they are wide characters (such as the + Unified Han Ideographs or Squared Katakana Symbols). This + category includes characters that have explicit halfwidth + counterparts. + Na 1 Narrow: characters that are always narrow and have explicit + fullwidth or wide counterparts. These characters are + implicitly narrow in East Asian typography and legacy + character sets because they have explicit fullwidth or wide + counterparts. All of ASCII is an example of East Asian + Narrow characters. + A 1 or 2 Ambiguous: characters that may sometimes be wide and + sometimes narrow. Ambiguous characters require additional + information not contained in the character code to further + resolve their width. This information is usually defined in + terminal setting that should in turn respect glyphs widths + in used fonts. Also see :ref:`ambiwidth configuration + option <config-common-ambiwidth>`. + N 1 Neutral characters: character that does not occur in legacy + East Asian character sets. + === ====== =========================================================== + + :param unicode string: + String whose width will be calculated. + + :return: unsigned integer.''') + + +def strwidth_ucs_4(width_data, string): + return sum((( + ( + 0 + ) if combining(symbol) else ( + width_data[east_asian_width(symbol)] + ) + ) for symbol in string)) + + +strwidth_ucs_4.__doc__ = _strwidth_documentation.format( + '''This version of function expects that characters above 0xFFFF are + represented using one symbol. This is only the case in UCS-4 Python builds. + + .. note: + Even in UCS-4 Python builds it is possible to represent characters above + 0xFFFF using surrogate pairs. Characters represented this way are not + supported.''') + + +def strwidth_ucs_2(width_data, string): + return sum((( + ( + width_data[east_asian_width(string[i - 1] + symbol)] + ) if 0xDC00 <= ord(symbol) <= 0xDFFF else ( + 0 + ) if combining(symbol) or 0xD800 <= ord(symbol) <= 0xDBFF else ( + width_data[east_asian_width(symbol)] + ) + ) for i, symbol in enumerate(string))) + + +strwidth_ucs_2.__doc__ = _strwidth_documentation.format( + '''This version of function expects that characters above 0xFFFF are + represented using two symbols forming a surrogate pair, which is the only + option in UCS-2 Python builds. It still works correctly in UCS-4 Python + builds, but is slower then its UCS-4 counterpart.''') diff --git a/powerline/lib/url.py b/powerline/lib/url.py new file mode 100644 index 0000000..f25919c --- /dev/null +++ b/powerline/lib/url.py @@ -0,0 +1,17 @@ +# vim:fileencoding=utf-8:noet +from __future__ import (unicode_literals, division, absolute_import, print_function) + +try: + from urllib.error import HTTPError # NOQA + from urllib.request import urlopen # NOQA + from urllib.parse import urlencode as urllib_urlencode # NOQA +except ImportError: + from urllib2 import urlopen, HTTPError # NOQA + from urllib import urlencode as urllib_urlencode # NOQA + + +def urllib_read(url): + try: + return urlopen(url, timeout=10).read().decode('utf-8') + except HTTPError: + return diff --git a/powerline/lib/vcs/__init__.py b/powerline/lib/vcs/__init__.py new file mode 100644 index 0000000..f862c6b --- /dev/null +++ b/powerline/lib/vcs/__init__.py @@ -0,0 +1,276 @@ +# vim:fileencoding=utf-8:noet +from __future__ import (unicode_literals, division, absolute_import, print_function) + +import os +import errno + +from threading import Lock +from collections import defaultdict + +from powerline.lib.watcher import create_tree_watcher +from powerline.lib.unicode import out_u +from powerline.lib.path import join + + +def generate_directories(path): + if os.path.isdir(path): + yield path + while True: + if os.path.ismount(path): + break + old_path = path + path = os.path.dirname(path) + if path == old_path or not path: + break + yield path + + +_file_watcher = None + + +def file_watcher(create_watcher): + global _file_watcher + if _file_watcher is None: + _file_watcher = create_watcher() + return _file_watcher + + +_branch_watcher = None + + +def branch_watcher(create_watcher): + global _branch_watcher + if _branch_watcher is None: + _branch_watcher = create_watcher() + return _branch_watcher + + +branch_name_cache = {} +branch_lock = Lock() +file_status_lock = Lock() + + +def get_branch_name(directory, config_file, get_func, create_watcher): + global branch_name_cache + with branch_lock: + # Check if the repo directory was moved/deleted + fw = branch_watcher(create_watcher) + is_watched = fw.is_watching(directory) + try: + changed = fw(directory) + except OSError as e: + if getattr(e, 'errno', None) != errno.ENOENT: + raise + changed = True + if changed: + branch_name_cache.pop(config_file, None) + # Remove the watches for this repo + if is_watched: + fw.unwatch(directory) + fw.unwatch(config_file) + else: + # Check if the config file has changed + try: + changed = fw(config_file) + except OSError as e: + if getattr(e, 'errno', None) != errno.ENOENT: + raise + # Config file does not exist (happens for mercurial) + if config_file not in branch_name_cache: + branch_name_cache[config_file] = out_u(get_func(directory, config_file)) + if changed: + # Config file has changed or was not tracked + branch_name_cache[config_file] = out_u(get_func(directory, config_file)) + return branch_name_cache[config_file] + + +class FileStatusCache(dict): + def __init__(self): + self.dirstate_map = defaultdict(set) + self.ignore_map = defaultdict(set) + self.keypath_ignore_map = {} + + def update_maps(self, keypath, directory, dirstate_file, ignore_file_name, extra_ignore_files): + parent = keypath + ignore_files = set() + while parent != directory: + nparent = os.path.dirname(keypath) + if nparent == parent: + break + parent = nparent + ignore_files.add(join(parent, ignore_file_name)) + for f in extra_ignore_files: + ignore_files.add(f) + self.keypath_ignore_map[keypath] = ignore_files + for ignf in ignore_files: + self.ignore_map[ignf].add(keypath) + self.dirstate_map[dirstate_file].add(keypath) + + def invalidate(self, dirstate_file=None, ignore_file=None): + for keypath in self.dirstate_map[dirstate_file]: + self.pop(keypath, None) + for keypath in self.ignore_map[ignore_file]: + self.pop(keypath, None) + + def ignore_files(self, keypath): + for ignf in self.keypath_ignore_map[keypath]: + yield ignf + + +file_status_cache = FileStatusCache() + + +def get_file_status(directory, dirstate_file, file_path, ignore_file_name, get_func, create_watcher, extra_ignore_files=()): + global file_status_cache + keypath = file_path if os.path.isabs(file_path) else join(directory, file_path) + file_status_cache.update_maps(keypath, directory, dirstate_file, ignore_file_name, extra_ignore_files) + + with file_status_lock: + # Optimize case of keypath not being cached + if keypath not in file_status_cache: + file_status_cache[keypath] = ans = get_func(directory, file_path) + return ans + + # Check if any relevant files have changed + file_changed = file_watcher(create_watcher) + changed = False + # Check if dirstate has changed + try: + changed = file_changed(dirstate_file) + except OSError as e: + if getattr(e, 'errno', None) != errno.ENOENT: + raise + # The .git index file does not exist for a new git repo + return get_func(directory, file_path) + + if changed: + # Remove all cached values for files that depend on this + # dirstate_file + file_status_cache.invalidate(dirstate_file=dirstate_file) + else: + # Check if the file itself has changed + try: + changed ^= file_changed(keypath) + except OSError as e: + if getattr(e, 'errno', None) != errno.ENOENT: + raise + # Do not call get_func again for a non-existent file + if keypath not in file_status_cache: + file_status_cache[keypath] = get_func(directory, file_path) + return file_status_cache[keypath] + + if changed: + file_status_cache.pop(keypath, None) + else: + # Check if one of the ignore files has changed + for ignf in file_status_cache.ignore_files(keypath): + try: + changed ^= file_changed(ignf) + except OSError as e: + if getattr(e, 'errno', None) != errno.ENOENT: + raise + if changed: + # Invalidate cache for all files that might be affected + # by this ignore file + file_status_cache.invalidate(ignore_file=ignf) + break + + try: + return file_status_cache[keypath] + except KeyError: + file_status_cache[keypath] = ans = get_func(directory, file_path) + return ans + + +class TreeStatusCache(dict): + def __init__(self, pl): + self.tw = create_tree_watcher(pl) + self.pl = pl + + def cache_and_get(self, key, status): + ans = self.get(key, self) + if ans is self: + ans = self[key] = status() + return ans + + def __call__(self, repo): + key = repo.directory + try: + if self.tw(key, ignore_event=getattr(repo, 'ignore_event', None)): + self.pop(key, None) + except OSError as e: + self.pl.warn('Failed to check {0} for changes, with error: {1}', key, str(e)) + return self.cache_and_get(key, repo.status) + + +_tree_status_cache = None + + +def tree_status(repo, pl): + global _tree_status_cache + if _tree_status_cache is None: + _tree_status_cache = TreeStatusCache(pl) + return _tree_status_cache(repo) + + +vcs_props = ( + ('git', '.git', os.path.exists), + ('mercurial', '.hg', os.path.isdir), + ('bzr', '.bzr', os.path.isdir), +) + + +vcs_props_bytes = [ + (vcs, vcs_dir.encode('ascii'), check) + for vcs, vcs_dir, check in vcs_props +] + + +def guess(path, create_watcher): + for directory in generate_directories(path): + for vcs, vcs_dir, check in (vcs_props_bytes if isinstance(path, bytes) else vcs_props): + repo_dir = os.path.join(directory, vcs_dir) + if check(repo_dir): + if os.path.isdir(repo_dir) and not os.access(repo_dir, os.X_OK): + continue + try: + if vcs not in globals(): + globals()[vcs] = getattr(__import__(str('powerline.lib.vcs'), fromlist=[str(vcs)]), str(vcs)) + return globals()[vcs].Repository(directory, create_watcher) + except: + pass + return None + + +def get_fallback_create_watcher(): + from powerline.lib.watcher import create_file_watcher + from powerline import get_fallback_logger + from functools import partial + return partial(create_file_watcher, get_fallback_logger(), 'auto') + + +def debug(): + '''Test run guess(), repo.branch() and repo.status() + + To use:: + python -c 'from powerline.lib.vcs import debug; debug()' some_file_to_watch. + ''' + import sys + dest = sys.argv[-1] + repo = guess(os.path.abspath(dest), get_fallback_create_watcher) + if repo is None: + print ('%s is not a recognized vcs repo' % dest) + raise SystemExit(1) + print ('Watching %s' % dest) + print ('Press Ctrl-C to exit.') + try: + while True: + if os.path.isdir(dest): + print ('Branch name: %s Status: %s' % (repo.branch(), repo.status())) + else: + print ('File status: %s' % repo.status(dest)) + raw_input('Press Enter to check again: ') + except KeyboardInterrupt: + pass + except EOFError: + pass diff --git a/powerline/lib/vcs/bzr.py b/powerline/lib/vcs/bzr.py new file mode 100644 index 0000000..e47d8b2 --- /dev/null +++ b/powerline/lib/vcs/bzr.py @@ -0,0 +1,108 @@ +# vim:fileencoding=utf-8:noet +from __future__ import (unicode_literals, division, absolute_import, print_function) + +import os +import re + +from io import StringIO + +from bzrlib import (workingtree, status, library_state, trace, ui) + +from powerline.lib.vcs import get_branch_name, get_file_status +from powerline.lib.path import join +from powerline.lib.encoding import get_preferred_file_contents_encoding + + +class CoerceIO(StringIO): + def write(self, arg): + if isinstance(arg, bytes): + arg = arg.decode(get_preferred_file_contents_encoding(), 'replace') + return super(CoerceIO, self).write(arg) + + +nick_pat = re.compile(br'nickname\s*=\s*(.+)') + + +def branch_name_from_config_file(directory, config_file): + ans = None + try: + with open(config_file, 'rb') as f: + for line in f: + m = nick_pat.match(line) + if m is not None: + ans = m.group(1).strip().decode(get_preferred_file_contents_encoding(), 'replace') + break + except Exception: + pass + return ans or os.path.basename(directory) + + +state = None + + +class Repository(object): + def __init__(self, directory, create_watcher): + self.directory = os.path.abspath(directory) + self.create_watcher = create_watcher + + def status(self, path=None): + '''Return status of repository or file. + + Without file argument: returns status of the repository: + + :'D?': dirty (tracked modified files: added, removed, deleted, modified), + :'?U': untracked-dirty (added, but not tracked files) + :None: clean (status is empty) + + With file argument: returns status of this file: The status codes are + those returned by bzr status -S + ''' + if path is not None: + return get_file_status( + directory=self.directory, + dirstate_file=join(self.directory, '.bzr', 'checkout', 'dirstate'), + file_path=path, + ignore_file_name='.bzrignore', + get_func=self.do_status, + create_watcher=self.create_watcher, + ) + return self.do_status(self.directory, path) + + def do_status(self, directory, path): + try: + return self._status(self.directory, path) + except Exception: + pass + + def _status(self, directory, path): + global state + if state is None: + state = library_state.BzrLibraryState(ui=ui.SilentUIFactory, trace=trace.DefaultConfig()) + buf = CoerceIO() + w = workingtree.WorkingTree.open(directory) + status.show_tree_status(w, specific_files=[path] if path else None, to_file=buf, short=True) + raw = buf.getvalue() + if not raw.strip(): + return + if path: + ans = raw[:2] + if ans == 'I ': # Ignored + ans = None + return ans + dirtied = untracked = ' ' + for line in raw.splitlines(): + if len(line) > 1 and line[1] in 'ACDMRIN': + dirtied = 'D' + elif line and line[0] == '?': + untracked = 'U' + ans = dirtied + untracked + return ans if ans.strip() else None + + def branch(self): + config_file = join(self.directory, '.bzr', 'branch', 'branch.conf') + return get_branch_name( + directory=self.directory, + config_file=config_file, + get_func=branch_name_from_config_file, + create_watcher=self.create_watcher, + ) diff --git a/powerline/lib/vcs/git.py b/powerline/lib/vcs/git.py new file mode 100644 index 0000000..bebc311 --- /dev/null +++ b/powerline/lib/vcs/git.py @@ -0,0 +1,208 @@ +# vim:fileencoding=utf-8:noet +from __future__ import (unicode_literals, division, absolute_import, print_function) + +import os +import re + +from powerline.lib.vcs import get_branch_name, get_file_status +from powerline.lib.shell import readlines +from powerline.lib.path import join +from powerline.lib.encoding import (get_preferred_file_name_encoding, + get_preferred_file_contents_encoding) +from powerline.lib.shell import which + + +_ref_pat = re.compile(br'ref:\s*refs/heads/(.+)') + + +def branch_name_from_config_file(directory, config_file): + try: + with open(config_file, 'rb') as f: + raw = f.read() + except EnvironmentError: + return os.path.basename(directory) + m = _ref_pat.match(raw) + if m is not None: + return m.group(1).decode(get_preferred_file_contents_encoding(), 'replace') + return raw[:7] + + +def git_directory(directory): + path = join(directory, '.git') + if os.path.isfile(path): + with open(path, 'rb') as f: + raw = f.read() + if not raw.startswith(b'gitdir: '): + raise IOError('invalid gitfile format') + raw = raw[8:] + if raw[-1:] == b'\n': + raw = raw[:-1] + if not isinstance(path, bytes): + raw = raw.decode(get_preferred_file_name_encoding()) + if not raw: + raise IOError('no path in gitfile') + return os.path.abspath(os.path.join(directory, raw)) + else: + return path + + +class GitRepository(object): + __slots__ = ('directory', 'create_watcher') + + def __init__(self, directory, create_watcher): + self.directory = os.path.abspath(directory) + self.create_watcher = create_watcher + + def status(self, path=None): + '''Return status of repository or file. + + Without file argument: returns status of the repository: + + :First column: working directory status (D: dirty / space) + :Second column: index status (I: index dirty / space) + :Third column: presence of untracked files (U: untracked files / space) + :None: repository clean + + With file argument: returns status of this file. Output is + equivalent to the first two columns of ``git status --porcelain`` + (except for merge statuses as they are not supported by libgit2). + ''' + if path: + gitd = git_directory(self.directory) + # We need HEAD as without it using fugitive to commit causes the + # current file’s status (and only the current file) to not be updated + # for some reason I cannot be bothered to figure out. + return get_file_status( + directory=self.directory, + dirstate_file=join(gitd, 'index'), + file_path=path, + ignore_file_name='.gitignore', + get_func=self.do_status, + create_watcher=self.create_watcher, + extra_ignore_files=tuple(join(gitd, x) for x in ('logs/HEAD', 'info/exclude')), + ) + return self.do_status(self.directory, path) + + def branch(self): + directory = git_directory(self.directory) + head = join(directory, 'HEAD') + return get_branch_name( + directory=directory, + config_file=head, + get_func=branch_name_from_config_file, + create_watcher=self.create_watcher, + ) + + +try: + import pygit2 as git + + class Repository(GitRepository): + @staticmethod + def ignore_event(path, name): + return False + + def stash(self): + try: + stashref = git.Repository(git_directory(self.directory)).lookup_reference('refs/stash') + except KeyError: + return 0 + return sum(1 for _ in stashref.log()) + + def do_status(self, directory, path): + if path: + try: + status = git.Repository(directory).status_file(path) + except (KeyError, ValueError): + return None + + if status == git.GIT_STATUS_CURRENT: + return None + else: + if status & git.GIT_STATUS_WT_NEW: + return '??' + if status & git.GIT_STATUS_IGNORED: + return '!!' + + if status & git.GIT_STATUS_INDEX_NEW: + index_status = 'A' + elif status & git.GIT_STATUS_INDEX_DELETED: + index_status = 'D' + elif status & git.GIT_STATUS_INDEX_MODIFIED: + index_status = 'M' + else: + index_status = ' ' + + if status & git.GIT_STATUS_WT_DELETED: + wt_status = 'D' + elif status & git.GIT_STATUS_WT_MODIFIED: + wt_status = 'M' + else: + wt_status = ' ' + + return index_status + wt_status + else: + wt_column = ' ' + index_column = ' ' + untracked_column = ' ' + for status in git.Repository(directory).status().values(): + if status & git.GIT_STATUS_WT_NEW: + untracked_column = 'U' + continue + + if status & (git.GIT_STATUS_WT_DELETED | git.GIT_STATUS_WT_MODIFIED): + wt_column = 'D' + + if status & ( + git.GIT_STATUS_INDEX_NEW + | git.GIT_STATUS_INDEX_MODIFIED + | git.GIT_STATUS_INDEX_DELETED + ): + index_column = 'I' + r = wt_column + index_column + untracked_column + return r if r != ' ' else None +except ImportError: + class Repository(GitRepository): + def __init__(self, *args, **kwargs): + if not which('git'): + raise OSError('git executable is not available') + super(Repository, self).__init__(*args, **kwargs) + + @staticmethod + def ignore_event(path, name): + # Ignore changes to the index.lock file, since they happen + # frequently and don't indicate an actual change in the working tree + # status + return path.endswith('.git') and name == 'index.lock' + + def _gitcmd(self, directory, *args): + return readlines(('git',) + args, directory) + + def stash(self): + return sum(1 for _ in self._gitcmd(self.directory, '--no-optional-locks', 'stash', 'list')) + + def do_status(self, directory, path): + if path: + try: + return next(self._gitcmd(directory, '--no-optional-locks', 'status', '--porcelain', '--ignored', '--', path))[:2] + except StopIteration: + return None + else: + wt_column = ' ' + index_column = ' ' + untracked_column = ' ' + for line in self._gitcmd(directory, '--no-optional-locks', 'status', '--porcelain'): + if line[0] == '?': + untracked_column = 'U' + continue + elif line[0] == '!': + continue + + if line[0] != ' ': + index_column = 'I' + + if line[1] != ' ': + wt_column = 'D' + + r = wt_column + index_column + untracked_column + return r if r != ' ' else None diff --git a/powerline/lib/vcs/mercurial.py b/powerline/lib/vcs/mercurial.py new file mode 100644 index 0000000..09b6e0b --- /dev/null +++ b/powerline/lib/vcs/mercurial.py @@ -0,0 +1,88 @@ +# vim:fileencoding=utf-8:noet +from __future__ import (unicode_literals, division, absolute_import, print_function) + +import os + +import hglib + +from powerline.lib.vcs import get_branch_name, get_file_status +from powerline.lib.path import join +from powerline.lib.encoding import get_preferred_file_contents_encoding + + +def branch_name_from_config_file(directory, config_file): + try: + with open(config_file, 'rb') as f: + raw = f.read() + return raw.decode(get_preferred_file_contents_encoding(), 'replace').strip() + except Exception: + return 'default' + + +class Repository(object): + __slots__ = ('directory', 'create_watcher') + + # hg status -> (powerline file status, repo status flag) + statuses = { + b'M': ('M', 1), b'A': ('A', 1), b'R': ('R', 1), b'!': ('D', 1), + b'?': ('U', 2), b'I': ('I', 0), b'C': ('', 0), + } + repo_statuses_str = (None, 'D ', ' U', 'DU') + + def __init__(self, directory, create_watcher): + self.directory = os.path.abspath(directory) + self.create_watcher = create_watcher + + def _repo(self, directory): + # Cannot create this object once and use always: when repository updates + # functions emit invalid results + return hglib.open(directory) + + def status(self, path=None): + '''Return status of repository or file. + + Without file argument: returns status of the repository: + + :'D?': dirty (tracked modified files: added, removed, deleted, modified), + :'?U': untracked-dirty (added, but not tracked files) + :None: clean (status is empty) + + With file argument: returns status of this file: `M`odified, `A`dded, + `R`emoved, `D`eleted (removed from filesystem, but still tracked), + `U`nknown, `I`gnored, (None)Clean. + ''' + if path: + return get_file_status( + directory=self.directory, + dirstate_file=join(self.directory, '.hg', 'dirstate'), + file_path=path, + ignore_file_name='.hgignore', + get_func=self.do_status, + create_watcher=self.create_watcher, + ) + return self.do_status(self.directory, path) + + def do_status(self, directory, path): + with self._repo(directory) as repo: + if path: + path = os.path.join(directory, path) + statuses = repo.status(include=path, all=True) + for status, paths in statuses: + if paths: + return self.statuses[status][0] + return None + else: + resulting_status = 0 + for status, paths in repo.status(all=True): + if paths: + resulting_status |= self.statuses[status][1] + return self.repo_statuses_str[resulting_status] + + def branch(self): + config_file = join(self.directory, '.hg', 'branch') + return get_branch_name( + directory=self.directory, + config_file=config_file, + get_func=branch_name_from_config_file, + create_watcher=self.create_watcher, + ) diff --git a/powerline/lib/watcher/__init__.py b/powerline/lib/watcher/__init__.py new file mode 100644 index 0000000..4fe9896 --- /dev/null +++ b/powerline/lib/watcher/__init__.py @@ -0,0 +1,76 @@ +# vim:fileencoding=utf-8:noet +from __future__ import (unicode_literals, division, absolute_import, print_function) + +import sys + +from powerline.lib.watcher.stat import StatFileWatcher +from powerline.lib.watcher.inotify import INotifyFileWatcher +from powerline.lib.watcher.tree import TreeWatcher +from powerline.lib.watcher.uv import UvFileWatcher, UvNotFound +from powerline.lib.inotify import INotifyError + + +def create_file_watcher(pl, watcher_type='auto', expire_time=10): + '''Create an object that can watch for changes to specified files + + Use ``.__call__()`` method of the returned object to start watching the file + or check whether file has changed since last call. + + Use ``.unwatch()`` method of the returned object to stop watching the file. + + Uses inotify if available, then pyuv, otherwise tracks mtimes. expire_time + is the number of minutes after the last query for a given path for the + inotify watch for that path to be automatically removed. This conserves + kernel resources. + + :param PowerlineLogger pl: + Logger. + :param str watcher_type + One of ``inotify`` (linux only), ``uv``, ``stat``, ``auto``. Determines + what watcher will be used. ``auto`` will use ``inotify`` if available, + then ``libuv`` and then fall back to ``stat``. + :param int expire_time: + Number of minutes since last ``.__call__()`` before inotify watcher will + stop watching given file. + ''' + if watcher_type == 'stat': + pl.debug('Using requested stat-based watcher', prefix='watcher') + return StatFileWatcher() + if watcher_type == 'inotify': + # Explicitly selected inotify watcher: do not catch INotifyError then. + pl.debug('Using requested inotify watcher', prefix='watcher') + return INotifyFileWatcher(expire_time=expire_time) + elif watcher_type == 'uv': + pl.debug('Using requested uv watcher', prefix='watcher') + return UvFileWatcher() + + if sys.platform.startswith('linux'): + try: + pl.debug('Trying to use inotify watcher', prefix='watcher') + return INotifyFileWatcher(expire_time=expire_time) + except INotifyError: + pl.info('Failed to create inotify watcher', prefix='watcher') + + try: + pl.debug('Using libuv-based watcher') + return UvFileWatcher() + except UvNotFound: + pl.debug('Failed to import pyuv') + + pl.debug('Using stat-based watcher') + return StatFileWatcher() + + +def create_tree_watcher(pl, watcher_type='auto', expire_time=10): + '''Create an object that can watch for changes in specified directories + + :param PowerlineLogger pl: + Logger. + :param str watcher_type: + Watcher type. Currently the only supported types are ``inotify`` (linux + only), ``uv``, ``dummy`` and ``auto``. + :param int expire_time: + Number of minutes since last ``.__call__()`` before inotify watcher will + stop watching given file. + ''' + return TreeWatcher(pl, watcher_type, expire_time) diff --git a/powerline/lib/watcher/inotify.py b/powerline/lib/watcher/inotify.py new file mode 100644 index 0000000..c4f1200 --- /dev/null +++ b/powerline/lib/watcher/inotify.py @@ -0,0 +1,268 @@ +# vim:fileencoding=utf-8:noet +from __future__ import (unicode_literals, division, absolute_import, print_function) + +import errno +import os +import ctypes + +from threading import RLock + +from powerline.lib.inotify import INotify +from powerline.lib.monotonic import monotonic +from powerline.lib.path import realpath + + +class INotifyFileWatcher(INotify): + def __init__(self, expire_time=10): + super(INotifyFileWatcher, self).__init__() + self.watches = {} + self.modified = {} + self.last_query = {} + self.lock = RLock() + self.expire_time = expire_time * 60 + + def expire_watches(self): + now = monotonic() + for path, last_query in tuple(self.last_query.items()): + if last_query - now > self.expire_time: + self.unwatch(path) + + def process_event(self, wd, mask, cookie, name): + if wd == -1 and (mask & self.Q_OVERFLOW): + # We missed some INOTIFY events, so we don't + # know the state of any tracked files. + for path in tuple(self.modified): + if os.path.exists(path): + self.modified[path] = True + else: + self.watches.pop(path, None) + self.modified.pop(path, None) + self.last_query.pop(path, None) + return + + for path, num in tuple(self.watches.items()): + if num == wd: + if mask & self.IGNORED: + self.watches.pop(path, None) + self.modified.pop(path, None) + self.last_query.pop(path, None) + else: + if mask & self.ATTRIB: + # The watched file could have had its inode changed, in + # which case we will not get any more events for this + # file, so re-register the watch. For example by some + # other file being renamed as this file. + try: + self.unwatch(path) + except OSError: + pass + try: + self.watch(path) + except OSError as e: + if getattr(e, 'errno', None) != errno.ENOENT: + raise + else: + self.modified[path] = True + else: + self.modified[path] = True + + def unwatch(self, path): + ''' Remove the watch for path. Raises an OSError if removing the watch + fails for some reason. ''' + path = realpath(path) + with self.lock: + self.modified.pop(path, None) + self.last_query.pop(path, None) + wd = self.watches.pop(path, None) + if wd is not None: + if self._rm_watch(self._inotify_fd, wd) != 0: + self.handle_error() + + def watch(self, path): + ''' Register a watch for the file/directory named path. Raises an OSError if path + does not exist. ''' + path = realpath(path) + with self.lock: + if path not in self.watches: + bpath = path if isinstance(path, bytes) else path.encode(self.fenc) + flags = self.MOVE_SELF | self.DELETE_SELF + buf = ctypes.c_char_p(bpath) + # Try watching path as a directory + wd = self._add_watch(self._inotify_fd, buf, flags | self.ONLYDIR) + if wd == -1: + eno = ctypes.get_errno() + if eno != errno.ENOTDIR: + self.handle_error() + # Try watching path as a file + flags |= (self.MODIFY | self.ATTRIB) + wd = self._add_watch(self._inotify_fd, buf, flags) + if wd == -1: + self.handle_error() + self.watches[path] = wd + self.modified[path] = False + + def is_watching(self, path): + with self.lock: + return realpath(path) in self.watches + + def __call__(self, path): + ''' Return True if path has been modified since the last call. Can + raise OSError if the path does not exist. ''' + path = realpath(path) + with self.lock: + self.last_query[path] = monotonic() + self.expire_watches() + if path not in self.watches: + # Try to re-add the watch, it will fail if the file does not + # exist/you don't have permission + self.watch(path) + return True + self.read(get_name=False) + if path not in self.modified: + # An ignored event was received which means the path has been + # automatically unwatched + return True + ans = self.modified[path] + if ans: + self.modified[path] = False + return ans + + def close(self): + with self.lock: + for path in tuple(self.watches): + try: + self.unwatch(path) + except OSError: + pass + super(INotifyFileWatcher, self).close() + + +class NoSuchDir(ValueError): + pass + + +class BaseDirChanged(ValueError): + pass + + +class DirTooLarge(ValueError): + def __init__(self, bdir): + ValueError.__init__(self, 'The directory {0} is too large to monitor. Try increasing the value in /proc/sys/fs/inotify/max_user_watches'.format(bdir)) + + +class INotifyTreeWatcher(INotify): + is_dummy = False + + def __init__(self, basedir, ignore_event=None): + super(INotifyTreeWatcher, self).__init__() + self.basedir = realpath(basedir) + self.watch_tree() + self.modified = True + self.ignore_event = (lambda path, name: False) if ignore_event is None else ignore_event + + def watch_tree(self): + self.watched_dirs = {} + self.watched_rmap = {} + try: + self.add_watches(self.basedir) + except OSError as e: + if e.errno == errno.ENOSPC: + raise DirTooLarge(self.basedir) + + def add_watches(self, base, top_level=True): + ''' Add watches for this directory and all its descendant directories, + recursively. ''' + base = realpath(base) + # There may exist a link which leads to an endless + # add_watches loop or to maximum recursion depth exceeded + if not top_level and base in self.watched_dirs: + return + try: + is_dir = self.add_watch(base) + except OSError as e: + if e.errno == errno.ENOENT: + # The entry could have been deleted between listdir() and + # add_watch(). + if top_level: + raise NoSuchDir('The dir {0} does not exist'.format(base)) + return + if e.errno == errno.EACCES: + # We silently ignore entries for which we don't have permission, + # unless they are the top level dir + if top_level: + raise NoSuchDir('You do not have permission to monitor {0}'.format(base)) + return + raise + else: + if is_dir: + try: + files = os.listdir(base) + except OSError as e: + if e.errno in (errno.ENOTDIR, errno.ENOENT): + # The dir was deleted/replaced between the add_watch() + # and listdir() + if top_level: + raise NoSuchDir('The dir {0} does not exist'.format(base)) + return + raise + for x in files: + self.add_watches(os.path.join(base, x), top_level=False) + elif top_level: + # The top level dir is a file, not good. + raise NoSuchDir('The dir {0} does not exist'.format(base)) + + def add_watch(self, path): + bpath = path if isinstance(path, bytes) else path.encode(self.fenc) + wd = self._add_watch( + self._inotify_fd, + ctypes.c_char_p(bpath), + + # Ignore symlinks and watch only directories + self.DONT_FOLLOW | self.ONLYDIR | + + self.MODIFY | self.CREATE | self.DELETE | + self.MOVE_SELF | self.MOVED_FROM | self.MOVED_TO | + self.ATTRIB | self.DELETE_SELF + ) + if wd == -1: + eno = ctypes.get_errno() + if eno == errno.ENOTDIR: + return False + raise OSError(eno, 'Failed to add watch for: {0}: {1}'.format(path, self.os.strerror(eno))) + self.watched_dirs[path] = wd + self.watched_rmap[wd] = path + return True + + def process_event(self, wd, mask, cookie, name): + if wd == -1 and (mask & self.Q_OVERFLOW): + # We missed some INOTIFY events, so we don't + # know the state of any tracked dirs. + self.watch_tree() + self.modified = True + return + path = self.watched_rmap.get(wd, None) + if path is not None: + if not self.ignore_event(path, name): + self.modified = True + if mask & self.CREATE: + # A new sub-directory might have been created, monitor it. + try: + if not isinstance(path, bytes): + name = name.decode(self.fenc) + self.add_watch(os.path.join(path, name)) + except OSError as e: + if e.errno == errno.ENOENT: + # Deleted before add_watch() + pass + elif e.errno == errno.ENOSPC: + raise DirTooLarge(self.basedir) + else: + raise + if (mask & self.DELETE_SELF or mask & self.MOVE_SELF) and path == self.basedir: + raise BaseDirChanged('The directory %s was moved/deleted' % path) + + def __call__(self): + self.read() + ret = self.modified + self.modified = False + return ret diff --git a/powerline/lib/watcher/stat.py b/powerline/lib/watcher/stat.py new file mode 100644 index 0000000..0c08971 --- /dev/null +++ b/powerline/lib/watcher/stat.py @@ -0,0 +1,44 @@ +# vim:fileencoding=utf-8:noet +from __future__ import (unicode_literals, division, absolute_import, print_function) + +import os + +from threading import RLock + +from powerline.lib.path import realpath + + +class StatFileWatcher(object): + def __init__(self): + self.watches = {} + self.lock = RLock() + + def watch(self, path): + path = realpath(path) + with self.lock: + self.watches[path] = os.path.getmtime(path) + + def unwatch(self, path): + path = realpath(path) + with self.lock: + self.watches.pop(path, None) + + def is_watching(self, path): + with self.lock: + return realpath(path) in self.watches + + def __call__(self, path): + path = realpath(path) + with self.lock: + if path not in self.watches: + self.watches[path] = os.path.getmtime(path) + return True + mtime = os.path.getmtime(path) + if mtime != self.watches[path]: + self.watches[path] = mtime + return True + return False + + def close(self): + with self.lock: + self.watches.clear() diff --git a/powerline/lib/watcher/tree.py b/powerline/lib/watcher/tree.py new file mode 100644 index 0000000..7d2b83f --- /dev/null +++ b/powerline/lib/watcher/tree.py @@ -0,0 +1,90 @@ +# vim:fileencoding=utf-8:noet +from __future__ import (unicode_literals, division, absolute_import, print_function) + +import sys + +from powerline.lib.monotonic import monotonic +from powerline.lib.inotify import INotifyError +from powerline.lib.path import realpath +from powerline.lib.watcher.inotify import INotifyTreeWatcher, DirTooLarge, NoSuchDir, BaseDirChanged +from powerline.lib.watcher.uv import UvTreeWatcher, UvNotFound + + +class DummyTreeWatcher(object): + is_dummy = True + + def __init__(self, basedir): + self.basedir = realpath(basedir) + + def __call__(self): + return False + + +class TreeWatcher(object): + def __init__(self, pl, watcher_type, expire_time): + self.watches = {} + self.last_query_times = {} + self.expire_time = expire_time * 60 + self.pl = pl + self.watcher_type = watcher_type + + def get_watcher(self, path, ignore_event): + if self.watcher_type == 'inotify': + return INotifyTreeWatcher(path, ignore_event=ignore_event) + if self.watcher_type == 'uv': + return UvTreeWatcher(path, ignore_event=ignore_event) + if self.watcher_type == 'dummy': + return DummyTreeWatcher(path) + # FIXME + if self.watcher_type == 'stat': + return DummyTreeWatcher(path) + if self.watcher_type == 'auto': + if sys.platform.startswith('linux'): + try: + return INotifyTreeWatcher(path, ignore_event=ignore_event) + except (INotifyError, DirTooLarge) as e: + if not isinstance(e, INotifyError): + self.pl.warn('Failed to watch path: {0} with error: {1}'.format(path, e)) + try: + return UvTreeWatcher(path, ignore_event=ignore_event) + except UvNotFound: + pass + return DummyTreeWatcher(path) + else: + raise ValueError('Unknown watcher type: {0}'.format(self.watcher_type)) + + def watch(self, path, ignore_event=None): + path = realpath(path) + w = self.get_watcher(path, ignore_event) + self.watches[path] = w + return w + + def expire_old_queries(self): + pop = [] + now = monotonic() + for path, lt in self.last_query_times.items(): + if now - lt > self.expire_time: + pop.append(path) + for path in pop: + del self.last_query_times[path] + + def __call__(self, path, ignore_event=None): + path = realpath(path) + self.expire_old_queries() + self.last_query_times[path] = monotonic() + w = self.watches.get(path, None) + if w is None: + try: + self.watch(path, ignore_event=ignore_event) + except NoSuchDir: + pass + return True + try: + return w() + except BaseDirChanged: + self.watches.pop(path, None) + return True + except DirTooLarge as e: + self.pl.warn(str(e)) + self.watches[path] = DummyTreeWatcher(path) + return False diff --git a/powerline/lib/watcher/uv.py b/powerline/lib/watcher/uv.py new file mode 100644 index 0000000..272db0f --- /dev/null +++ b/powerline/lib/watcher/uv.py @@ -0,0 +1,207 @@ +# vim:fileencoding=utf-8:noet +from __future__ import (unicode_literals, division, absolute_import, print_function) + +import os + +from collections import defaultdict +from threading import RLock +from functools import partial +from threading import Thread +from errno import ENOENT + +from powerline.lib.path import realpath +from powerline.lib.encoding import get_preferred_file_name_encoding + + +class UvNotFound(NotImplementedError): + pass + + +pyuv = None +pyuv_version_info = None + + +def import_pyuv(): + global pyuv + global pyuv_version_info + if not pyuv: + try: + pyuv = __import__('pyuv') + except ImportError: + raise UvNotFound + else: + pyuv_version_info = tuple((int(c) for c in pyuv.__version__.split('.'))) + + +class UvThread(Thread): + daemon = True + + def __init__(self, loop): + self.uv_loop = loop + self.async_handle = pyuv.Async(loop, self._async_cb) + super(UvThread, self).__init__() + + def _async_cb(self, handle): + self.uv_loop.stop() + self.async_handle.close() + + def run(self): + self.uv_loop.run() + + def join(self): + self.async_handle.send() + return super(UvThread, self).join() + + +_uv_thread = None + + +def start_uv_thread(): + global _uv_thread + if _uv_thread is None: + loop = pyuv.Loop() + _uv_thread = UvThread(loop) + _uv_thread.start() + return _uv_thread.uv_loop + + +def normpath(path, fenc): + path = realpath(path) + if isinstance(path, bytes): + return path.decode(fenc) + else: + return path + + +class UvWatcher(object): + def __init__(self): + import_pyuv() + self.watches = {} + self.lock = RLock() + self.loop = start_uv_thread() + self.fenc = get_preferred_file_name_encoding() + if pyuv_version_info >= (1, 0): + self._start_watch = self._start_watch_1_x + else: + self._start_watch = self._start_watch_0_x + + def _start_watch_1_x(self, path): + handle = pyuv.fs.FSEvent(self.loop) + handle.start(path, 0, partial(self._record_event, path)) + self.watches[path] = handle + + def _start_watch_0_x(self, path): + self.watches[path] = pyuv.fs.FSEvent( + self.loop, + path, + partial(self._record_event, path), + pyuv.fs.UV_CHANGE | pyuv.fs.UV_RENAME + ) + + def watch(self, path): + path = normpath(path, self.fenc) + with self.lock: + if path not in self.watches: + try: + self._start_watch(path) + except pyuv.error.FSEventError as e: + code = e.args[0] + if code == pyuv.errno.UV_ENOENT: + raise OSError(ENOENT, 'No such file or directory: ' + path) + else: + raise + + def unwatch(self, path): + path = normpath(path, self.fenc) + with self.lock: + try: + watch = self.watches.pop(path) + except KeyError: + return + watch.close(partial(self._stopped_watching, path)) + + def is_watching(self, path): + with self.lock: + return normpath(path, self.fenc) in self.watches + + def __del__(self): + try: + lock = self.lock + except AttributeError: + pass + else: + with lock: + while self.watches: + path, watch = self.watches.popitem() + watch.close(partial(self._stopped_watching, path)) + + +class UvFileWatcher(UvWatcher): + def __init__(self): + super(UvFileWatcher, self).__init__() + self.events = defaultdict(list) + + def _record_event(self, path, fsevent_handle, filename, events, error): + with self.lock: + self.events[path].append(events) + if events | pyuv.fs.UV_RENAME: + if not os.path.exists(path): + self.watches.pop(path).close() + + def _stopped_watching(self, path, *args): + self.events.pop(path, None) + + def __call__(self, path): + path = normpath(path, self.fenc) + with self.lock: + events = self.events.pop(path, None) + if events: + return True + if path not in self.watches: + self.watch(path) + return True + return False + + +class UvTreeWatcher(UvWatcher): + is_dummy = False + + def __init__(self, basedir, ignore_event=None): + super(UvTreeWatcher, self).__init__() + self.ignore_event = ignore_event or (lambda path, name: False) + self.basedir = normpath(basedir, self.fenc) + self.modified = True + self.watch_directory(self.basedir) + + def watch_directory(self, path): + for root, dirs, files in os.walk(normpath(path, self.fenc)): + self.watch_one_directory(root) + + def watch_one_directory(self, dirname): + try: + self.watch(dirname) + except OSError: + pass + + def _stopped_watching(self, path, *args): + pass + + def _record_event(self, path, fsevent_handle, filename, events, error): + if not self.ignore_event(path, filename): + self.modified = True + if events == pyuv.fs.UV_CHANGE | pyuv.fs.UV_RENAME: + # Stat changes to watched directory are UV_CHANGE|UV_RENAME. It + # is weird. + pass + elif events | pyuv.fs.UV_RENAME: + if not os.path.isdir(path): + self.unwatch(path) + else: + full_name = os.path.join(path, filename) + if os.path.isdir(full_name): + # For some reason mkdir and rmdir both fall into this + # category + self.watch_directory(full_name) + + def __call__(self): + return self.__dict__.pop('modified', False) |