Merge pyextra subtree

pull/819/head^2
Vehicle Researcher 2018-12-10 14:06:10 -08:00
commit 573a6915fc
8 changed files with 209 additions and 312 deletions

View File

@ -1,123 +0,0 @@
Metadata-Version: 1.1
Name: overpy
Version: 0.4
Summary: Python Wrapper to access the OpenStreepMap Overpass API
Home-page: https://github.com/DinoTools/python-overpy
Author: PhiBo (DinoTools)
Author-email: UNKNOWN
License: MIT
Description: Python Overpass Wrapper
=======================
A Python Wrapper to access the Overpass API.
Have a look at the `documentation`_ to find additional information.
.. image:: https://pypip.in/version/overpy/badge.svg
:target: https://pypi.python.org/pypi/overpy/
:alt: Latest Version
.. image:: https://pypip.in/license/overpy/badge.svg
:target: https://pypi.python.org/pypi/overpy/
:alt: License
.. image:: https://travis-ci.org/DinoTools/python-overpy.svg?branch=master
:target: https://travis-ci.org/DinoTools/python-overpy
.. image:: https://coveralls.io/repos/DinoTools/python-overpy/badge.png?branch=master
:target: https://coveralls.io/r/DinoTools/python-overpy?branch=master
Features
--------
* Query Overpass API
* Parse JSON and XML response data
* Additional helper functions
Install
-------
**Requirements:**
Supported Python versions:
* Python 2.7
* Python >= 3.2
* PyPy and PyPy3
**Install:**
.. code-block:: console
$ pip install overpy
Examples
--------
Additional examples can be found in the `documentation`_ and in the *examples* directory.
.. code-block:: python
import overpy
api = overpy.Overpass()
# fetch all ways and nodes
result = api.query("""
way(50.746,7.154,50.748,7.157) ["highway"];
(._;>;);
out body;
""")
for way in result.ways:
print("Name: %s" % way.tags.get("name", "n/a"))
print(" Highway: %s" % way.tags.get("highway", "n/a"))
print(" Nodes:")
for node in way.nodes:
print(" Lat: %f, Lon: %f" % (node.lat, node.lon))
Helper
~~~~~~
Helper methods are available to provide easy access to often used requests.
.. code-block:: python
import overpy.helper
# 3600062594 is the OSM id of Chemnitz and is the bounding box for the request
street = overpy.helper.get_street(
"Straße der Nationen",
"3600062594"
)
# this finds an intersection between Straße der Nationen and Carolastraße in Chemnitz
intersection = overpy.helper.get_intersection(
"Straße der Nationen",
"Carolastraße",
"3600062594"
)
License
-------
Published under the MIT (see LICENSE for more information)
.. _`documentation`: http://python-overpy.readthedocs.org/
Keywords: OverPy Overpass OSM OpenStreetMap
Platform: UNKNOWN
Classifier: Development Status :: 4 - Beta
Classifier: License :: OSI Approved :: MIT License
Classifier: Operating System :: OS Independent
Classifier: Programming Language :: Python
Classifier: Programming Language :: Python :: 2.7
Classifier: Programming Language :: Python :: 3
Classifier: Programming Language :: Python :: 3.2
Classifier: Programming Language :: Python :: 3.3
Classifier: Programming Language :: Python :: 3.4
Classifier: Programming Language :: Python :: 3.5
Classifier: Programming Language :: Python :: Implementation :: CPython
Classifier: Programming Language :: Python :: Implementation :: PyPy

View File

@ -1,61 +0,0 @@
CHANGELOG.rst
LICENSE
MANIFEST.in
README.rst
setup.cfg
setup.py
docs/make.bat
docs/source/api.rst
docs/source/changelog.rst
docs/source/conf.py
docs/source/contributing.rst
docs/source/example.rst
docs/source/index.rst
docs/source/introduction.rst
examples/get_areas.py
examples/get_nodes.py
examples/get_ways.py
overpy/__about__.py
overpy/__init__.py
overpy/exception.py
overpy/helper.py
overpy.egg-info/PKG-INFO
overpy.egg-info/SOURCES.txt
overpy.egg-info/dependency_links.txt
overpy.egg-info/not-zip-safe
overpy.egg-info/top_level.txt
tests/__init__.py
tests/base_class.py
tests/test_exception.py
tests/test_json.py
tests/test_request.py
tests/test_result.py
tests/test_result_way.py
tests/test_xml.py
tests/json/area-01.json
tests/json/node-01.json
tests/json/relation-01.json
tests/json/relation-02.json
tests/json/relation-03.json
tests/json/relation-04.json
tests/json/result-expand-01.json
tests/json/result-expand-02.json
tests/json/result-way-01.json
tests/json/result-way-02.json
tests/json/result-way-03.json
tests/json/way-01.json
tests/json/way-02.json
tests/json/way-03.json
tests/json/way-04.json
tests/response/bad-request-encoding.html
tests/response/bad-request.html
tests/xml/area-01.xml
tests/xml/node-01.xml
tests/xml/relation-01.xml
tests/xml/relation-02.xml
tests/xml/relation-03.xml
tests/xml/relation-04.xml
tests/xml/way-01.xml
tests/xml/way-02.xml
tests/xml/way-03.xml
tests/xml/way-04.xml

View File

@ -1,56 +0,0 @@
../overpy/__about__.py
../overpy/__about__.pyc
../overpy/__init__.py
../overpy/__init__.pyc
../overpy/exception.py
../overpy/exception.pyc
../overpy/helper.py
../overpy/helper.pyc
../tests/__init__.py
../tests/__init__.pyc
../tests/base_class.py
../tests/base_class.pyc
../tests/json/area-01.json
../tests/json/node-01.json
../tests/json/relation-01.json
../tests/json/relation-02.json
../tests/json/relation-03.json
../tests/json/relation-04.json
../tests/json/result-expand-01.json
../tests/json/result-expand-02.json
../tests/json/result-way-01.json
../tests/json/result-way-02.json
../tests/json/result-way-03.json
../tests/json/way-01.json
../tests/json/way-02.json
../tests/json/way-03.json
../tests/json/way-04.json
../tests/response/bad-request-encoding.html
../tests/response/bad-request.html
../tests/test_exception.py
../tests/test_exception.pyc
../tests/test_json.py
../tests/test_json.pyc
../tests/test_request.py
../tests/test_request.pyc
../tests/test_result.py
../tests/test_result.pyc
../tests/test_result_way.py
../tests/test_result_way.pyc
../tests/test_xml.py
../tests/test_xml.pyc
../tests/xml/area-01.xml
../tests/xml/node-01.xml
../tests/xml/relation-01.xml
../tests/xml/relation-02.xml
../tests/xml/relation-03.xml
../tests/xml/relation-04.xml
../tests/xml/way-01.xml
../tests/xml/way-02.xml
../tests/xml/way-03.xml
../tests/xml/way-04.xml
PKG-INFO
SOURCES.txt
dependency_links.txt
not-zip-safe
top_level.txt

View File

@ -1,2 +0,0 @@
overpy
tests

View File

@ -5,6 +5,8 @@ from xml.sax import handler, make_parser
import json
import re
import sys
import time
import requests
from overpy import exception
from overpy.__about__ import (
@ -18,12 +20,15 @@ PY3 = sys.version_info[0] == 3
XML_PARSER_DOM = 1
XML_PARSER_SAX = 2
if PY2:
from urllib2 import urlopen
from urllib2 import HTTPError
elif PY3:
from urllib.request import urlopen
from urllib.error import HTTPError
# Try to convert some common attributes
# http://wiki.openstreetmap.org/wiki/Elements#Common_attributes
GLOBAL_ATTRIBUTE_MODIFIERS = {
"changeset": int,
"timestamp": lambda ts: datetime.strptime(ts, "%Y-%m-%dT%H:%M:%SZ"),
"uid": int,
"version": int,
"visible": lambda v: v.lower() == "true"
}
def is_valid_type(element, cls):
@ -41,11 +46,16 @@ def is_valid_type(element, cls):
class Overpass(object):
"""
Class to access the Overpass API
:cvar default_max_retry_count: Global max number of retries (Default: 0)
:cvar default_retry_timeout: Global time to wait between tries (Default: 1.0s)
"""
default_max_retry_count = 0
default_read_chunk_size = 4096
default_retry_timeout = 1.0
default_url = "http://overpass-api.de/api/interpreter"
def __init__(self, read_chunk_size=None, url=None, xml_parser=XML_PARSER_SAX):
def __init__(self, read_chunk_size=None, url=None, xml_parser=XML_PARSER_SAX, max_retry_count=None, retry_timeout=None, timeout=5.0, headers=None):
"""
:param read_chunk_size: Max size of each chunk read from the server response
:type read_chunk_size: Integer
@ -53,6 +63,14 @@ class Overpass(object):
:type url: str
:param xml_parser: The xml parser to use
:type xml_parser: Integer
:param max_retry_count: Max number of retries (Default: default_max_retry_count)
:type max_retry_count: Integer
:param retry_timeout: Time to wait between tries (Default: default_retry_timeout)
:type retry_timeout: float
:param timeout: HTTP request timeout
:type timeout: float
:param headers: HTTP request headers
:type headers: dict
"""
self.url = self.default_url
if url is not None:
@ -63,7 +81,34 @@ class Overpass(object):
if read_chunk_size is None:
read_chunk_size = self.default_read_chunk_size
self.read_chunk_size = read_chunk_size
if max_retry_count is None:
max_retry_count = self.default_max_retry_count
self.max_retry_count = max_retry_count
if retry_timeout is None:
retry_timeout = self.default_retry_timeout
self.retry_timeout = retry_timeout
self.xml_parser = xml_parser
self.timeout = timeout
self.headers = headers
def _handle_remark_msg(self, msg):
"""
Try to parse the message provided with the remark tag or element.
:param str msg: The message
:raises overpy.exception.OverpassRuntimeError: If message starts with 'runtime error:'
:raises overpy.exception.OverpassRuntimeRemark: If message starts with 'runtime remark:'
:raises overpy.exception.OverpassUnknownError: If we are unable to identify the error
"""
msg = msg.strip()
if msg.startswith("runtime error:"):
raise exception.OverpassRuntimeError(msg=msg)
elif msg.startswith("runtime remark:"):
raise exception.OverpassRuntimeRemark(msg=msg)
raise exception.OverpassUnknownError(msg=msg)
def query(self, query):
"""
@ -76,56 +121,79 @@ class Overpass(object):
if not isinstance(query, bytes):
query = query.encode("utf-8")
try:
f = urlopen(self.url, query)
except HTTPError as e:
f = e
retry_num = 0
retry_exceptions = []
do_retry = True if self.max_retry_count > 0 else False
while retry_num <= self.max_retry_count:
if retry_num > 0:
time.sleep(self.retry_timeout)
retry_num += 1
response = f.read(self.read_chunk_size)
while True:
data = f.read(self.read_chunk_size)
if len(data) == 0:
break
response = response + data
f.close()
try:
if self.headers is not None:
r = requests.post(self.url, query, timeout=self.timeout, headers=self.headers)
else:
r = requests.post(self.url, query, timeout=self.timeout)
response = r.content
except (requests.exceptions.BaseHTTPError, requests.exceptions.RequestException) as e:
if not do_retry:
raise e
retry_exceptions.append(e)
continue
if f.code == 200:
if PY2:
http_info = f.info()
content_type = http_info.getheader("content-type")
else:
content_type = f.getheader("Content-Type")
if r.status_code == 200:
content_type = r.headers["Content-Type"]
if content_type == "application/json":
return self.parse_json(response)
if content_type == "application/json":
return self.parse_json(response)
if content_type == "application/osm3s+xml":
return self.parse_xml(response)
if content_type == "application/osm3s+xml":
return self.parse_xml(response)
raise exception.OverpassUnknownContentType(content_type)
e = exception.OverpassUnknownContentType(content_type)
if not do_retry:
raise e
retry_exceptions.append(e)
continue
elif r.status_code == 400:
msgs = []
for msg in self._regex_extract_error_msg.finditer(response):
tmp = self._regex_remove_tag.sub(b"", msg.group("msg"))
try:
tmp = tmp.decode("utf-8")
except UnicodeDecodeError:
tmp = repr(tmp)
msgs.append(tmp)
if f.code == 400:
msgs = []
for msg in self._regex_extract_error_msg.finditer(response):
tmp = self._regex_remove_tag.sub(b"", msg.group("msg"))
try:
tmp = tmp.decode("utf-8")
except UnicodeDecodeError:
tmp = repr(tmp)
msgs.append(tmp)
e = exception.OverpassBadRequest(
query,
msgs=msgs
)
if not do_retry:
raise e
retry_exceptions.append(e)
continue
elif r.status_code == 429:
e = exception.OverpassTooManyRequests
if not do_retry:
raise e
retry_exceptions.append(e)
continue
elif r.status_code == 504:
e = exception.OverpassGatewayTimeout
if not do_retry:
raise e
retry_exceptions.append(e)
continue
raise exception.OverpassBadRequest(
query,
msgs=msgs
)
# No valid response code
e = exception.OverpassUnknownHTTPStatusCode(r.status_code)
if not do_retry:
raise e
retry_exceptions.append(e)
continue
if f.code == 429:
raise exception.OverpassTooManyRequests
if f.code == 504:
raise exception.OverpassGatewayTimeout
raise exception.OverpassUnknownHTTPStatusCode(f.code)
raise exception.MaxRetriesReached(retry_count=retry_num, exceptions=retry_exceptions)
def parse_json(self, data, encoding="utf-8"):
"""
@ -139,8 +207,11 @@ class Overpass(object):
:rtype: overpy.Result
"""
if isinstance(data, bytes):
data = data.decode(encoding)
data = data.decode(encoding)
data = json.loads(data, parse_float=Decimal)
if "remark" in data:
self._handle_remark_msg(msg=data.get("remark"))
return Result.from_json(data, api=self)
def parse_xml(self, data, encoding="utf-8", parser=None):
@ -155,13 +226,16 @@ class Overpass(object):
"""
if parser is None:
parser = self.xml_parser
if isinstance(data, bytes):
data = data.decode(encoding)
if PY2 and not isinstance(data, str):
# Python 2.x: Convert unicode strings
data = data.encode(encoding)
m = re.compile("<remark>(?P<msg>[^<>]*)</remark>").search(data)
if m:
self._handle_remark_msg(m.group("msg"))
return Result.from_xml(data, api=self, parser=parser)
@ -279,23 +353,39 @@ class Result(object):
return result
@classmethod
def from_xml(cls, data, api=None, parser=XML_PARSER_SAX):
def from_xml(cls, data, api=None, parser=None):
"""
Create a new instance and load data from xml object.
Create a new instance and load data from xml data or object.
.. note::
If parser is set to None, the functions tries to find the best parse.
By default the SAX parser is chosen if a string is provided as data.
The parser is set to DOM if an xml.etree.ElementTree.Element is provided as data value.
:param data: Root element
:type data: xml.etree.ElementTree.Element
:param api:
:type data: str | xml.etree.ElementTree.Element
:param api: The instance to query additional information if required.
:type api: Overpass
:param parser: Specify the parser to use(DOM or SAX)
:type parser: Integer
:param parser: Specify the parser to use(DOM or SAX)(Default: None = autodetect, defaults to SAX)
:type parser: Integer | None
:return: New instance of Result object
:rtype: Result
"""
if parser is None:
if isinstance(data, str):
parser = XML_PARSER_SAX
else:
parser = XML_PARSER_DOM
result = cls(api=api)
if parser == XML_PARSER_DOM:
import xml.etree.ElementTree as ET
root = ET.fromstring(data)
if isinstance(data, str):
root = ET.fromstring(data)
elif isinstance(data, ET.Element):
root = data
else:
raise exception.OverPyException("Unable to detect data type.")
for elem_cls in [Node, Way, Relation, Area]:
for child in root:
@ -522,17 +612,10 @@ class Element(object):
"""
self._result = result
# Try to convert some common attributes
# http://wiki.openstreetmap.org/wiki/Elements#Common_attributes
self._attribute_modifiers = {
"changeset": int,
"timestamp": lambda ts: datetime.strptime(ts, "%Y-%m-%dT%H:%M:%SZ"),
"uid": int,
"version": int,
"visible": lambda v: v.lower() == "true"
}
self.attributes = attributes
for n, m in self._attribute_modifiers.items():
# ToDo: Add option to modify attribute modifiers
attribute_modifiers = dict(GLOBAL_ATTRIBUTE_MODIFIERS.items())
for n, m in attribute_modifiers.items():
if n in self.attributes:
self.attributes[n] = m(self.attributes[n])
self.id = None

View File

@ -37,6 +37,18 @@ class ElementDataWrongType(OverPyException):
)
class MaxRetriesReached(OverPyException):
"""
Raised if max retries reached and the Overpass server didn't respond with a result.
"""
def __init__(self, retry_count, exceptions):
self.exceptions = exceptions
self.retry_count = retry_count
def __str__(self):
return "Unable get any result from the Overpass API server after %d retries." % self.retry_count
class OverpassBadRequest(OverPyException):
"""
Raised if the Overpass API service returns a syntax error.
@ -62,6 +74,29 @@ class OverpassBadRequest(OverPyException):
return "\n".join(tmp_msgs)
class OverpassError(OverPyException):
"""
Base exception to report errors if the response returns a remark tag or element.
.. note::
If you are not sure which of the subexceptions you should use, use this one and try to parse the message.
For more information have a look at https://github.com/DinoTools/python-overpy/issues/62
:param str msg: The message from the remark tag or element
"""
def __init__(self, msg=None):
#: The message from the remark tag or element
self.msg = msg
def __str__(self):
if self.msg is None:
return "No error message provided"
if not isinstance(self.msg, str):
return str(self.msg)
return self.msg
class OverpassGatewayTimeout(OverPyException):
"""
Raised if load of the Overpass API service is too high and it can't handle the request.
@ -70,6 +105,22 @@ class OverpassGatewayTimeout(OverPyException):
OverPyException.__init__(self, "Server load too high")
class OverpassRuntimeError(OverpassError):
"""
Raised if the server returns a remark-tag(xml) or remark element(json) with a message starting with
'runtime error:'.
"""
pass
class OverpassRuntimeRemark(OverpassError):
"""
Raised if the server returns a remark-tag(xml) or remark element(json) with a message starting with
'runtime remark:'.
"""
pass
class OverpassTooManyRequests(OverPyException):
"""
Raised if the Overpass API service returns a 429 status code.
@ -94,6 +145,13 @@ class OverpassUnknownContentType(OverPyException):
return "Unknown content type: %s" % self.content_type
class OverpassUnknownError(OverpassError):
"""
Raised if the server returns a remark-tag(xml) or remark element(json) and we are unable to find any reason.
"""
pass
class OverpassUnknownHTTPStatusCode(OverPyException):
"""
Raised if the returned HTTP status code isn't handled by OverPy.