diff --git a/pyextra/logentries/__init__.py b/pyextra/logentries/__init__.py deleted file mode 100644 index b64e423b..00000000 --- a/pyextra/logentries/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .utils import LogentriesHandler diff --git a/pyextra/logentries/helpers.py b/pyextra/logentries/helpers.py deleted file mode 100644 index bec676e2..00000000 --- a/pyextra/logentries/helpers.py +++ /dev/null @@ -1,49 +0,0 @@ - -""" This file contains some helpers methods in both Python2 and 3 """ -import sys -import re - -if sys.version < '3': - # Python2.x imports - import Queue - import codecs -else: - # Python 3.x imports - import queue - - -def check_token(token): - """ Checks if the given token is a valid UUID.""" - valid = re.compile(r"^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-" - r"[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$") - - return valid.match(token) - -# We need to do some things different pending if its Python 2.x or 3.x -if sys.version < '3': - def to_unicode(ch): - return codecs.unicode_escape_decode(ch)[0] - - def is_unicode(ch): - return isinstance(ch, unicode) - - def create_unicode(ch): - try: - return unicode(ch, 'utf-8') - except UnicodeDecodeError as e: - return str(e) - - def create_queue(max_size): - return Queue.Queue(max_size) -else: - def to_unicode(ch): - return ch - - def is_unicode(ch): - return isinstance(ch, str) - - def create_unicode(ch): - return str(ch) - - def create_queue(max_size): - return queue.Queue(max_size) diff --git a/pyextra/logentries/metrics.py b/pyextra/logentries/metrics.py deleted file mode 100644 index 03ddb643..00000000 --- a/pyextra/logentries/metrics.py +++ /dev/null @@ -1,57 +0,0 @@ -from logentries import LogentriesHandler -from threading import Lock -from functools import wraps -import logging -import time -import sys -import psutil - -glob_time = 0 -glob_name = 0 - -log = logging.getLogger('logentries') -log.setLevel(logging.INFO) - -class Metric(object): - - def __init__(self, token): - self._count = 0.0 - self._sum = 0.0 - self._lock = Lock() - self.token = token - handler = LogentriesHandler(token) - log.addHandler(handler) - - def observe(self, amount): - with self._lock: - self._count += 1 - self._sum += amount - - def metric(self): - '''Mesaure function execution time in seconds - and forward it to Logentries''' - - class Timer(object): - - def __init__(self, summary): - self._summary = summary - - def __enter__(self): - self._start = time.time() - - def __exit__(self, typ, value, traceback): - global glob_time - self._summary.observe(max(time.time() - self._start, 0)) - glob_time = time.time()- self._start - log.info("function_name=" + glob_name + " " + "execution_time=" + str(glob_time) + " " + "cpu=" + str(psutil.cpu_percent(interval=None)) + " " + "cpu_count=" + str(psutil.cpu_count())+ " " + "memory=" + str(psutil.virtual_memory()) ) - - def __call__(self, f): - @wraps(f) - def wrapped(*args, **kwargs): - with self: - global glob_name - glob_name = f.__name__ - - return f(*args, **kwargs) - return wrapped - return Timer(self) diff --git a/pyextra/logentries/utils.py b/pyextra/logentries/utils.py deleted file mode 100644 index c17a1070..00000000 --- a/pyextra/logentries/utils.py +++ /dev/null @@ -1,218 +0,0 @@ -# coding: utf-8 -# vim: set ts=4 sw=4 et: -""" This file contains some utils for connecting to Logentries - as well as storing logs in a queue and sending them.""" - -VERSION = '2.0.7' - -from logentries import helpers as le_helpers - -import logging -import threading -import socket -import random -import time -import sys - -import certifi - - -# Size of the internal event queue -QUEUE_SIZE = 32768 -# Logentries API server address -LE_API_DEFAULT = "data.logentries.com" -# Port number for token logging to Logentries API server -LE_PORT_DEFAULT = 80 -LE_TLS_PORT_DEFAULT = 443 -# Minimal delay between attempts to reconnect in seconds -MIN_DELAY = 0.1 -# Maximal delay between attempts to recconect in seconds -MAX_DELAY = 10 -# Unicode Line separator character \u2028 -LINE_SEP = le_helpers.to_unicode('\u2028') - - -# LE appender signature - used for debugging messages -LE = "LE: " -# Error message displayed when an incorrect Token has been detected -INVALID_TOKEN = ("\n\nIt appears the LOGENTRIES_TOKEN " - "parameter you entered is incorrect!\n\n") - - -def dbg(msg): - print(LE + msg) - - -class PlainTextSocketAppender(threading.Thread): - def __init__(self, verbose=True, le_api=LE_API_DEFAULT, le_port=LE_PORT_DEFAULT, le_tls_port=LE_TLS_PORT_DEFAULT): - threading.Thread.__init__(self) - - # Logentries API server address - self.le_api = le_api - - # Port number for token logging to Logentries API server - self.le_port = le_port - self.le_tls_port = le_tls_port - - self.daemon = True - self.verbose = verbose - self._conn = None - self._queue = le_helpers.create_queue(QUEUE_SIZE) - - def empty(self): - return self._queue.empty() - - def open_connection(self): - self._conn = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - self._conn.connect((self.le_api, self.le_port)) - - def reopen_connection(self): - self.close_connection() - - root_delay = MIN_DELAY - while True: - try: - self.open_connection() - return - except Exception: - if self.verbose: - dbg("Unable to connect to Logentries") - - root_delay *= 2 - if(root_delay > MAX_DELAY): - root_delay = MAX_DELAY - - wait_for = root_delay + random.uniform(0, root_delay) - - try: - time.sleep(wait_for) - except KeyboardInterrupt: - raise - - def close_connection(self): - if self._conn is not None: - self._conn.close() - - def run(self): - try: - # Open connection - self.reopen_connection() - - # Send data in queue - while True: - # Take data from queue - data = self._queue.get(block=True) - - # Replace newlines with Unicode line separator - # for multi-line events - if not le_helpers.is_unicode(data): - multiline = le_helpers.create_unicode(data).replace( - '\n', LINE_SEP) - else: - multiline = data.replace('\n', LINE_SEP) - multiline += "\n" - # Send data, reconnect if needed - while True: - try: - self._conn.send(multiline.encode('utf-8')) - except socket.error: - self.reopen_connection() - continue - break - except KeyboardInterrupt: - if self.verbose: - dbg("Logentries asynchronous socket client interrupted") - - self.close_connection() - -SocketAppender = PlainTextSocketAppender - -try: - import ssl - ssl_enabled = True -except ImportError: # for systems without TLS support. - ssl_enabled = False - dbg("Unable to import ssl module. Will send over port 80.") -else: - class TLSSocketAppender(PlainTextSocketAppender): - - def open_connection(self): - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sock = ssl.wrap_socket( - sock=sock, - keyfile=None, - certfile=None, - server_side=False, - cert_reqs=ssl.CERT_REQUIRED, - ssl_version=getattr( - ssl, - 'PROTOCOL_TLSv1_2', - ssl.PROTOCOL_TLSv1 - ), - ca_certs=certifi.where(), - do_handshake_on_connect=True, - suppress_ragged_eofs=True, - ) - - sock.connect((self.le_api, self.le_tls_port)) - self._conn = sock - - -class LogentriesHandler(logging.Handler): - def __init__(self, token, use_tls=True, verbose=True, format=None, le_api=LE_API_DEFAULT, le_port=LE_PORT_DEFAULT, le_tls_port=LE_TLS_PORT_DEFAULT): - logging.Handler.__init__(self) - self.token = token - self.good_config = True - self.verbose = verbose - # give the socket 10 seconds to flush, - # otherwise drop logs - self.timeout = 10 - if not le_helpers.check_token(token): - if self.verbose: - dbg(INVALID_TOKEN) - self.good_config = False - if format is None: - format = logging.Formatter('%(asctime)s : %(levelname)s, %(message)s', - '%a %b %d %H:%M:%S %Z %Y') - self.setFormatter(format) - self.setLevel(logging.DEBUG) - if use_tls and ssl_enabled: - self._thread = TLSSocketAppender(verbose=verbose, le_api=le_api, le_port=le_port, le_tls_port=le_tls_port) - else: - self._thread = SocketAppender(verbose=verbose, le_api=le_api, le_port=le_port, le_tls_port=le_tls_port) - - def flush(self): - # wait for all queued logs to be send - now = time.time() - while not self._thread.empty(): - time.sleep(0.2) - if time.time() - now > self.timeout: - break - - def emit_raw(self, msg): - if self.good_config and not self._thread.is_alive(): - try: - self._thread.start() - if self.verbose: - dbg("Starting Logentries Asynchronous Socket Appender") - except RuntimeError: # It's already started. - pass - - msg = self.token + msg - try: - self._thread._queue.put_nowait(msg) - except Exception: - # Queue is full, try to remove the oldest message and put again - try: - self._thread._queue.get_nowait() - self._thread._queue.put_nowait(msg) - except Exception: - # Race condition, no need for any action here - pass - - def emit(self, record): - msg = self.format(record).rstrip('\n') - self.emit_raw(msg) - - def close(self): - logging.Handler.close(self) diff --git a/release/files_common b/release/files_common index 9728a0c4..1732476b 100644 --- a/release/files_common +++ b/release/files_common @@ -426,7 +426,7 @@ installer/updater/Makefile scripts/update_now.sh scripts/stop_updater.sh -pyextra/logentries/** +pyextra/.gitignore rednose/**