merge in pyextra
George Hotz
parent
29ac3da7b8
commit
3fe9bbe665
|
|
@ -0,0 +1 @@
|
|||
*.pyc
|
||||
|
|
@ -0,0 +1 @@
|
|||
from .utils import LogentriesHandler
|
||||
|
|
@ -0,0 +1,49 @@
|
|||
|
||||
""" This file contains some helpers methods in both Python2 and 3 """
|
||||
import sys
|
||||
import re
|
||||
|
||||
if sys.version < '3':
|
||||
# Python2.x imports
|
||||
import Queue
|
||||
import codecs
|
||||
else:
|
||||
# Python 3.x imports
|
||||
import queue
|
||||
|
||||
|
||||
def check_token(token):
|
||||
""" Checks if the given token is a valid UUID."""
|
||||
valid = re.compile(r"^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-"
|
||||
r"[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$")
|
||||
|
||||
return valid.match(token)
|
||||
|
||||
# We need to do some things different pending if its Python 2.x or 3.x
|
||||
if sys.version < '3':
|
||||
def to_unicode(ch):
|
||||
return codecs.unicode_escape_decode(ch)[0]
|
||||
|
||||
def is_unicode(ch):
|
||||
return isinstance(ch, unicode)
|
||||
|
||||
def create_unicode(ch):
|
||||
try:
|
||||
return unicode(ch, 'utf-8')
|
||||
except UnicodeDecodeError as e:
|
||||
return str(e)
|
||||
|
||||
def create_queue(max_size):
|
||||
return Queue.Queue(max_size)
|
||||
else:
|
||||
def to_unicode(ch):
|
||||
return ch
|
||||
|
||||
def is_unicode(ch):
|
||||
return isinstance(ch, str)
|
||||
|
||||
def create_unicode(ch):
|
||||
return str(ch)
|
||||
|
||||
def create_queue(max_size):
|
||||
return queue.Queue(max_size)
|
||||
|
|
@ -0,0 +1,57 @@
|
|||
from logentries import LogentriesHandler
|
||||
from threading import Lock
|
||||
from functools import wraps
|
||||
import logging
|
||||
import time
|
||||
import sys
|
||||
import psutil
|
||||
|
||||
glob_time = 0
|
||||
glob_name = 0
|
||||
|
||||
log = logging.getLogger('logentries')
|
||||
log.setLevel(logging.INFO)
|
||||
|
||||
class Metric(object):
|
||||
|
||||
def __init__(self, token):
|
||||
self._count = 0.0
|
||||
self._sum = 0.0
|
||||
self._lock = Lock()
|
||||
self.token = token
|
||||
handler = LogentriesHandler(token)
|
||||
log.addHandler(handler)
|
||||
|
||||
def observe(self, amount):
|
||||
with self._lock:
|
||||
self._count += 1
|
||||
self._sum += amount
|
||||
|
||||
def metric(self):
|
||||
'''Mesaure function execution time in seconds
|
||||
and forward it to Logentries'''
|
||||
|
||||
class Timer(object):
|
||||
|
||||
def __init__(self, summary):
|
||||
self._summary = summary
|
||||
|
||||
def __enter__(self):
|
||||
self._start = time.time()
|
||||
|
||||
def __exit__(self, typ, value, traceback):
|
||||
global glob_time
|
||||
self._summary.observe(max(time.time() - self._start, 0))
|
||||
glob_time = time.time()- self._start
|
||||
log.info("function_name=" + glob_name + " " + "execution_time=" + str(glob_time) + " " + "cpu=" + str(psutil.cpu_percent(interval=None)) + " " + "cpu_count=" + str(psutil.cpu_count())+ " " + "memory=" + str(psutil.virtual_memory()) )
|
||||
|
||||
def __call__(self, f):
|
||||
@wraps(f)
|
||||
def wrapped(*args, **kwargs):
|
||||
with self:
|
||||
global glob_name
|
||||
glob_name = f.__name__
|
||||
|
||||
return f(*args, **kwargs)
|
||||
return wrapped
|
||||
return Timer(self)
|
||||
|
|
@ -0,0 +1,218 @@
|
|||
# coding: utf-8
|
||||
# vim: set ts=4 sw=4 et:
|
||||
""" This file contains some utils for connecting to Logentries
|
||||
as well as storing logs in a queue and sending them."""
|
||||
|
||||
VERSION = '2.0.7'
|
||||
|
||||
from logentries import helpers as le_helpers
|
||||
|
||||
import logging
|
||||
import threading
|
||||
import socket
|
||||
import random
|
||||
import time
|
||||
import sys
|
||||
|
||||
import certifi
|
||||
|
||||
|
||||
# Size of the internal event queue
|
||||
QUEUE_SIZE = 32768
|
||||
# Logentries API server address
|
||||
LE_API_DEFAULT = "data.logentries.com"
|
||||
# Port number for token logging to Logentries API server
|
||||
LE_PORT_DEFAULT = 80
|
||||
LE_TLS_PORT_DEFAULT = 443
|
||||
# Minimal delay between attempts to reconnect in seconds
|
||||
MIN_DELAY = 0.1
|
||||
# Maximal delay between attempts to recconect in seconds
|
||||
MAX_DELAY = 10
|
||||
# Unicode Line separator character \u2028
|
||||
LINE_SEP = le_helpers.to_unicode('\u2028')
|
||||
|
||||
|
||||
# LE appender signature - used for debugging messages
|
||||
LE = "LE: "
|
||||
# Error message displayed when an incorrect Token has been detected
|
||||
INVALID_TOKEN = ("\n\nIt appears the LOGENTRIES_TOKEN "
|
||||
"parameter you entered is incorrect!\n\n")
|
||||
|
||||
|
||||
def dbg(msg):
|
||||
print(LE + msg)
|
||||
|
||||
|
||||
class PlainTextSocketAppender(threading.Thread):
|
||||
def __init__(self, verbose=True, le_api=LE_API_DEFAULT, le_port=LE_PORT_DEFAULT, le_tls_port=LE_TLS_PORT_DEFAULT):
|
||||
threading.Thread.__init__(self)
|
||||
|
||||
# Logentries API server address
|
||||
self.le_api = le_api
|
||||
|
||||
# Port number for token logging to Logentries API server
|
||||
self.le_port = le_port
|
||||
self.le_tls_port = le_tls_port
|
||||
|
||||
self.daemon = True
|
||||
self.verbose = verbose
|
||||
self._conn = None
|
||||
self._queue = le_helpers.create_queue(QUEUE_SIZE)
|
||||
|
||||
def empty(self):
|
||||
return self._queue.empty()
|
||||
|
||||
def open_connection(self):
|
||||
self._conn = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
self._conn.connect((self.le_api, self.le_port))
|
||||
|
||||
def reopen_connection(self):
|
||||
self.close_connection()
|
||||
|
||||
root_delay = MIN_DELAY
|
||||
while True:
|
||||
try:
|
||||
self.open_connection()
|
||||
return
|
||||
except Exception:
|
||||
if self.verbose:
|
||||
dbg("Unable to connect to Logentries")
|
||||
|
||||
root_delay *= 2
|
||||
if(root_delay > MAX_DELAY):
|
||||
root_delay = MAX_DELAY
|
||||
|
||||
wait_for = root_delay + random.uniform(0, root_delay)
|
||||
|
||||
try:
|
||||
time.sleep(wait_for)
|
||||
except KeyboardInterrupt:
|
||||
raise
|
||||
|
||||
def close_connection(self):
|
||||
if self._conn is not None:
|
||||
self._conn.close()
|
||||
|
||||
def run(self):
|
||||
try:
|
||||
# Open connection
|
||||
self.reopen_connection()
|
||||
|
||||
# Send data in queue
|
||||
while True:
|
||||
# Take data from queue
|
||||
data = self._queue.get(block=True)
|
||||
|
||||
# Replace newlines with Unicode line separator
|
||||
# for multi-line events
|
||||
if not le_helpers.is_unicode(data):
|
||||
multiline = le_helpers.create_unicode(data).replace(
|
||||
'\n', LINE_SEP)
|
||||
else:
|
||||
multiline = data.replace('\n', LINE_SEP)
|
||||
multiline += "\n"
|
||||
# Send data, reconnect if needed
|
||||
while True:
|
||||
try:
|
||||
self._conn.send(multiline.encode('utf-8'))
|
||||
except socket.error:
|
||||
self.reopen_connection()
|
||||
continue
|
||||
break
|
||||
except KeyboardInterrupt:
|
||||
if self.verbose:
|
||||
dbg("Logentries asynchronous socket client interrupted")
|
||||
|
||||
self.close_connection()
|
||||
|
||||
SocketAppender = PlainTextSocketAppender
|
||||
|
||||
try:
|
||||
import ssl
|
||||
ssl_enabled = True
|
||||
except ImportError: # for systems without TLS support.
|
||||
ssl_enabled = False
|
||||
dbg("Unable to import ssl module. Will send over port 80.")
|
||||
else:
|
||||
class TLSSocketAppender(PlainTextSocketAppender):
|
||||
|
||||
def open_connection(self):
|
||||
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
sock = ssl.wrap_socket(
|
||||
sock=sock,
|
||||
keyfile=None,
|
||||
certfile=None,
|
||||
server_side=False,
|
||||
cert_reqs=ssl.CERT_REQUIRED,
|
||||
ssl_version=getattr(
|
||||
ssl,
|
||||
'PROTOCOL_TLSv1_2',
|
||||
ssl.PROTOCOL_TLSv1
|
||||
),
|
||||
ca_certs=certifi.where(),
|
||||
do_handshake_on_connect=True,
|
||||
suppress_ragged_eofs=True,
|
||||
)
|
||||
|
||||
sock.connect((self.le_api, self.le_tls_port))
|
||||
self._conn = sock
|
||||
|
||||
|
||||
class LogentriesHandler(logging.Handler):
|
||||
def __init__(self, token, use_tls=True, verbose=True, format=None, le_api=LE_API_DEFAULT, le_port=LE_PORT_DEFAULT, le_tls_port=LE_TLS_PORT_DEFAULT):
|
||||
logging.Handler.__init__(self)
|
||||
self.token = token
|
||||
self.good_config = True
|
||||
self.verbose = verbose
|
||||
# give the socket 10 seconds to flush,
|
||||
# otherwise drop logs
|
||||
self.timeout = 10
|
||||
if not le_helpers.check_token(token):
|
||||
if self.verbose:
|
||||
dbg(INVALID_TOKEN)
|
||||
self.good_config = False
|
||||
if format is None:
|
||||
format = logging.Formatter('%(asctime)s : %(levelname)s, %(message)s',
|
||||
'%a %b %d %H:%M:%S %Z %Y')
|
||||
self.setFormatter(format)
|
||||
self.setLevel(logging.DEBUG)
|
||||
if use_tls and ssl_enabled:
|
||||
self._thread = TLSSocketAppender(verbose=verbose, le_api=le_api, le_port=le_port, le_tls_port=le_tls_port)
|
||||
else:
|
||||
self._thread = SocketAppender(verbose=verbose, le_api=le_api, le_port=le_port, le_tls_port=le_tls_port)
|
||||
|
||||
def flush(self):
|
||||
# wait for all queued logs to be send
|
||||
now = time.time()
|
||||
while not self._thread.empty():
|
||||
time.sleep(0.2)
|
||||
if time.time() - now > self.timeout:
|
||||
break
|
||||
|
||||
def emit_raw(self, msg):
|
||||
if self.good_config and not self._thread.is_alive():
|
||||
try:
|
||||
self._thread.start()
|
||||
if self.verbose:
|
||||
dbg("Starting Logentries Asynchronous Socket Appender")
|
||||
except RuntimeError: # It's already started.
|
||||
pass
|
||||
|
||||
msg = self.token + msg
|
||||
try:
|
||||
self._thread._queue.put_nowait(msg)
|
||||
except Exception:
|
||||
# Queue is full, try to remove the oldest message and put again
|
||||
try:
|
||||
self._thread._queue.get_nowait()
|
||||
self._thread._queue.put_nowait(msg)
|
||||
except Exception:
|
||||
# Race condition, no need for any action here
|
||||
pass
|
||||
|
||||
def emit(self, record):
|
||||
msg = self.format(record).rstrip('\n')
|
||||
self.emit_raw(msg)
|
||||
|
||||
def close(self):
|
||||
logging.Handler.close(self)
|
||||
|
|
@ -0,0 +1,22 @@
|
|||
__all__ = [
|
||||
"__author__",
|
||||
"__copyright__",
|
||||
"__email__",
|
||||
"__license__",
|
||||
"__summary__",
|
||||
"__title__",
|
||||
"__uri__",
|
||||
"__version__",
|
||||
]
|
||||
|
||||
__title__ = "overpy"
|
||||
__summary__ = "Python Wrapper to access the OpenStreepMap Overpass API"
|
||||
__uri__ = "https://github.com/DinoTools/python-overpy"
|
||||
|
||||
__version__ = "0.4"
|
||||
|
||||
__author__ = "PhiBo (DinoTools)"
|
||||
__email__ = ""
|
||||
|
||||
__license__ = "MIT"
|
||||
__copyright__ = "Copyright 2014-2016 %s" % __author__
|
||||
File diff suppressed because it is too large
Load Diff
|
|
@ -0,0 +1,166 @@
|
|||
class OverPyException(BaseException):
|
||||
"""OverPy base exception"""
|
||||
pass
|
||||
|
||||
|
||||
class DataIncomplete(OverPyException):
|
||||
"""
|
||||
Raised if the requested data isn't available in the result.
|
||||
Try to improve the query or to resolve the missing data.
|
||||
"""
|
||||
def __init__(self, *args, **kwargs):
|
||||
OverPyException.__init__(
|
||||
self,
|
||||
"Data incomplete try to improve the query to resolve the missing data",
|
||||
*args,
|
||||
**kwargs
|
||||
)
|
||||
|
||||
|
||||
class ElementDataWrongType(OverPyException):
|
||||
"""
|
||||
Raised if the provided element does not match the expected type.
|
||||
|
||||
:param type_expected: The expected element type
|
||||
:type type_expected: String
|
||||
:param type_provided: The provided element type
|
||||
:type type_provided: String|None
|
||||
"""
|
||||
def __init__(self, type_expected, type_provided=None):
|
||||
self.type_expected = type_expected
|
||||
self.type_provided = type_provided
|
||||
|
||||
def __str__(self):
|
||||
return "Type expected '%s' but '%s' provided" % (
|
||||
self.type_expected,
|
||||
str(self.type_provided)
|
||||
)
|
||||
|
||||
|
||||
class MaxRetriesReached(OverPyException):
|
||||
"""
|
||||
Raised if max retries reached and the Overpass server didn't respond with a result.
|
||||
"""
|
||||
def __init__(self, retry_count, exceptions):
|
||||
self.exceptions = exceptions
|
||||
self.retry_count = retry_count
|
||||
|
||||
def __str__(self):
|
||||
return "Unable get any result from the Overpass API server after %d retries." % self.retry_count
|
||||
|
||||
|
||||
class OverpassBadRequest(OverPyException):
|
||||
"""
|
||||
Raised if the Overpass API service returns a syntax error.
|
||||
|
||||
:param query: The encoded query how it was send to the server
|
||||
:type query: Bytes
|
||||
:param msgs: List of error messages
|
||||
:type msgs: List
|
||||
"""
|
||||
def __init__(self, query, msgs=None):
|
||||
self.query = query
|
||||
if msgs is None:
|
||||
msgs = []
|
||||
self.msgs = msgs
|
||||
|
||||
def __str__(self):
|
||||
tmp_msgs = []
|
||||
for tmp_msg in self.msgs:
|
||||
if not isinstance(tmp_msg, str):
|
||||
tmp_msg = str(tmp_msg)
|
||||
tmp_msgs.append(tmp_msg)
|
||||
|
||||
return "\n".join(tmp_msgs)
|
||||
|
||||
|
||||
class OverpassError(OverPyException):
|
||||
"""
|
||||
Base exception to report errors if the response returns a remark tag or element.
|
||||
|
||||
.. note::
|
||||
If you are not sure which of the subexceptions you should use, use this one and try to parse the message.
|
||||
|
||||
For more information have a look at https://github.com/DinoTools/python-overpy/issues/62
|
||||
|
||||
:param str msg: The message from the remark tag or element
|
||||
"""
|
||||
def __init__(self, msg=None):
|
||||
#: The message from the remark tag or element
|
||||
self.msg = msg
|
||||
|
||||
def __str__(self):
|
||||
if self.msg is None:
|
||||
return "No error message provided"
|
||||
if not isinstance(self.msg, str):
|
||||
return str(self.msg)
|
||||
return self.msg
|
||||
|
||||
|
||||
class OverpassGatewayTimeout(OverPyException):
|
||||
"""
|
||||
Raised if load of the Overpass API service is too high and it can't handle the request.
|
||||
"""
|
||||
def __init__(self):
|
||||
OverPyException.__init__(self, "Server load too high")
|
||||
|
||||
|
||||
class OverpassRuntimeError(OverpassError):
|
||||
"""
|
||||
Raised if the server returns a remark-tag(xml) or remark element(json) with a message starting with
|
||||
'runtime error:'.
|
||||
"""
|
||||
pass
|
||||
|
||||
|
||||
class OverpassRuntimeRemark(OverpassError):
|
||||
"""
|
||||
Raised if the server returns a remark-tag(xml) or remark element(json) with a message starting with
|
||||
'runtime remark:'.
|
||||
"""
|
||||
pass
|
||||
|
||||
|
||||
class OverpassTooManyRequests(OverPyException):
|
||||
"""
|
||||
Raised if the Overpass API service returns a 429 status code.
|
||||
"""
|
||||
def __init__(self):
|
||||
OverPyException.__init__(self, "Too many requests")
|
||||
|
||||
|
||||
class OverpassUnknownContentType(OverPyException):
|
||||
"""
|
||||
Raised if the reported content type isn't handled by OverPy.
|
||||
|
||||
:param content_type: The reported content type
|
||||
:type content_type: None or String
|
||||
"""
|
||||
def __init__(self, content_type):
|
||||
self.content_type = content_type
|
||||
|
||||
def __str__(self):
|
||||
if self.content_type is None:
|
||||
return "No content type returned"
|
||||
return "Unknown content type: %s" % self.content_type
|
||||
|
||||
|
||||
class OverpassUnknownError(OverpassError):
|
||||
"""
|
||||
Raised if the server returns a remark-tag(xml) or remark element(json) and we are unable to find any reason.
|
||||
"""
|
||||
pass
|
||||
|
||||
|
||||
class OverpassUnknownHTTPStatusCode(OverPyException):
|
||||
"""
|
||||
Raised if the returned HTTP status code isn't handled by OverPy.
|
||||
|
||||
:param code: The HTTP status code
|
||||
:type code: Integer
|
||||
"""
|
||||
def __init__(self, code):
|
||||
self.code = code
|
||||
|
||||
def __str__(self):
|
||||
return "Unknown/Unhandled status code: %d" % self.code
|
||||
|
|
@ -0,0 +1,64 @@
|
|||
__author__ = 'mjob'
|
||||
|
||||
import overpy
|
||||
|
||||
|
||||
def get_street(street, areacode, api=None):
|
||||
"""
|
||||
Retrieve streets in a given bounding area
|
||||
|
||||
:param overpy.Overpass api: First street of intersection
|
||||
:param String street: Name of street
|
||||
:param String areacode: The OSM id of the bounding area
|
||||
:return: Parsed result
|
||||
:raises overpy.exception.OverPyException: If something bad happens.
|
||||
"""
|
||||
if api is None:
|
||||
api = overpy.Overpass()
|
||||
|
||||
query = """
|
||||
area(%s)->.location;
|
||||
(
|
||||
way[highway][name="%s"](area.location);
|
||||
- (
|
||||
way[highway=service](area.location);
|
||||
way[highway=track](area.location);
|
||||
);
|
||||
);
|
||||
out body;
|
||||
>;
|
||||
out skel qt;
|
||||
"""
|
||||
|
||||
data = api.query(query % (areacode, street))
|
||||
|
||||
return data
|
||||
|
||||
|
||||
def get_intersection(street1, street2, areacode, api=None):
|
||||
"""
|
||||
Retrieve intersection of two streets in a given bounding area
|
||||
|
||||
:param overpy.Overpass api: First street of intersection
|
||||
:param String street1: Name of first street of intersection
|
||||
:param String street2: Name of second street of intersection
|
||||
:param String areacode: The OSM id of the bounding area
|
||||
:return: List of intersections
|
||||
:raises overpy.exception.OverPyException: If something bad happens.
|
||||
"""
|
||||
if api is None:
|
||||
api = overpy.Overpass()
|
||||
|
||||
query = """
|
||||
area(%s)->.location;
|
||||
(
|
||||
way[highway][name="%s"](area.location); node(w)->.n1;
|
||||
way[highway][name="%s"](area.location); node(w)->.n2;
|
||||
);
|
||||
node.n1.n2;
|
||||
out meta;
|
||||
"""
|
||||
|
||||
data = api.query(query % (areacode, street1, street2))
|
||||
|
||||
return data.get_nodes()
|
||||
Loading…
Reference in New Issue