diff --git a/.flake8 b/.flake8
index 251b187..f038b2f 100644
--- a/.flake8
+++ b/.flake8
@@ -1,5 +1,7 @@
[flake8]
-ignore = E501,E711,I201,E731,W503,W504,W505
+ignore = E501,E731
+max-line-length = 120
+extend-ignore = E203,W503
exclude = ve*,.ve*,.eggs,node_modules,*_pb2.py,build
application-import-names = trackers
import-order-style = smarkets
\ No newline at end of file
diff --git a/checks b/checks
new file mode 100755
index 0000000..238fe55
--- /dev/null
+++ b/checks
@@ -0,0 +1,12 @@
+#!/bin/bash
+
+set -e
+
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+cd $SCRIPT_DIR
+
+set -x
+ve/bin/isort trackers
+ve/bin/black trackers --quiet
+ve/bin/flake8 trackers
+ve/bin/pytest trackers
\ No newline at end of file
diff --git a/pyproject.toml b/pyproject.toml
new file mode 100644
index 0000000..eb1c60c
--- /dev/null
+++ b/pyproject.toml
@@ -0,0 +1,75 @@
+[project]
+name = "trackers"
+readme = "README.md"
+dependencies = [
+ "aioauth-client",
+ "aiohttp",
+ "aiohttp_session[secure]",
+ "aiomsgpack",
+ "asyncinotify",
+ "aniso8601",
+ "arsenic",
+ "beautifulsoup4",
+ "calmjs.parse",
+ "dulwich",
+ "dulwich-tree",
+ "fixtures",
+ "geographiclib",
+ "htmlwrite",
+ "jsonpointer",
+ "libsass",
+ "more-itertools",
+ "msgpack",
+ "numpy",
+ "nvector==0.7.6",
+ "polyline",
+ "python-datauri",
+ "python-dateutil",
+ "python-slugify",
+ "pyyaml",
+ "tap.py",
+ "testresources",
+ "testscenarios",
+ "uvloop",
+]
+
+[project.optional-dependencies]
+dev = [
+ "flake8",
+ "black",
+ "isort",
+ "pytest",
+ "pytest-asyncio",
+]
+
+[project.scripts]
+add_gpx_to_event_routes = "trackers.bin_utils:add_gpx_to_event_routes"
+assign_rider_colors = "trackers.bin_utils:assign_rider_colors"
+convert_to_static = "trackers.bin_utils:convert_to_static"
+gen_key = "trackers.auth:gen_key"
+load_riders_from_csv = "trackers.bin_utils:load_riders_from_csv"
+print_names_and_trackers = "trackers.bin_utils:print_names_and_trackers"
+process_event_routes = "trackers.bin_utils:process_event_routes"
+reformat_event = "trackers.bin_utils:reformat_event"
+run_analyse = "trackers.bin_utils:analyse"
+run_qunit_tests = "trackers.client_test_tools:qunit_runner"
+serve = "trackers.serve:main"
+store_analyse = "trackers.bin_utils:store_analyse"
+trackleaders_get_config = "trackers.sources.trackleaders:get_config"
+update_bounds = "trackers.bin_utils:update_bounds"
+
+[tool.pdm]
+
+[build-system]
+requires = ["pdm-pep517>=1.0.0"]
+build-backend = "pdm.pep517.api"
+
+[tool.black]
+line-length = 100
+
+[tool.isort]
+profile = "black"
+
+[tool.pytest.ini_options]
+asyncio_mode = "auto"
+addopts = "--tb=short"
diff --git a/setup.py b/setup.py
deleted file mode 100644
index d7201d0..0000000
--- a/setup.py
+++ /dev/null
@@ -1,63 +0,0 @@
-import setuptools
-
-
-setuptools.setup(
- name='trackers',
- packages=setuptools.find_packages(),
- include_package_data=True,
- install_requires=[
- 'aioauth-client',
- 'aiohttp',
- 'aiohttp_session[secure]',
- 'aiomsgpack',
- 'aionotify',
- 'aniso8601',
- 'arsenic',
- 'asynctest',
- 'beautifulsoup4',
- 'calmjs.parse',
- 'dulwich',
- 'fixtures',
- 'htmlwrite',
- 'geographiclib',
- 'jsonpointer',
- 'libsass',
- 'more-itertools',
- 'msgpack',
- 'nvector==0.7.6',
- 'numpy',
- 'polyline',
- 'python-datauri',
- 'python-dateutil',
- 'python-slugify',
- 'pyyaml',
- 'tap.py',
- 'testresources',
- 'testscenarios',
- 'uvloop',
- ],
- entry_points={
- 'console_scripts': [
- 'serve=trackers.serve:main',
-
- # Tools to edit events
- 'convert_to_static=trackers.bin_utils:convert_to_static',
- 'store_analyse=trackers.bin_utils:store_analyse',
-
- 'assign_rider_colors=trackers.bin_utils:assign_rider_colors',
- 'add_gpx_to_event_routes=trackers.bin_utils:add_gpx_to_event_routes',
- 'reformat_event=trackers.bin_utils:reformat_event',
- 'process_event_routes=trackers.bin_utils:process_event_routes',
- 'update_bounds=trackers.bin_utils:update_bounds',
- 'load_riders_from_csv=trackers.bin_utils:load_riders_from_csv',
- 'run_analyse=trackers.bin_utils:analyse',
- 'print_names_and_trackers=trackers.bin_utils:print_names_and_trackers',
-
- 'run_qunit_tests=trackers.client_test_tools:qunit_runner',
- 'gen_key=trackers.auth:gen_key',
-
- 'trackleaders_get_config=trackers.sources.trackleaders:get_config',
- ],
- },
- test_suite='trackers.tests.suite',
-)
diff --git a/trackers/analyse.py b/trackers/analyse.py
index b291504..c0ef659 100644
--- a/trackers/analyse.py
+++ b/trackers/analyse.py
@@ -12,41 +12,36 @@
from typing import Any
from more_itertools import spy, take
-from numpy import (
- arccos,
- cross,
- deg2rad,
- dot,
- rad2deg,
- seterr,
-)
+from numpy import arccos, cross, deg2rad, dot, rad2deg, seterr
from numpy.linalg import norm
-from nvector import (
- interpolate,
- lat_lon2n_E,
- n_E2lat_lon,
- n_EB_E2p_EB_E,
- unit,
-)
+from nvector import interpolate, lat_lon2n_E, n_E2lat_lon, n_EB_E2p_EB_E, unit
-from trackers.base import cancel_and_wait_task, general_fut_done_callback, Observable, Tracker
+from trackers.base import (
+ Observable,
+ Tracker,
+ cancel_and_wait_task,
+ general_fut_done_callback,
+)
from trackers.contrib.dataclass_tools import add_slots
logger = logging.getLogger(__name__)
-seterr(all='raise')
+seterr(all="raise")
class AnalyseTracker(Tracker):
-
@classmethod
- async def start(cls, org_tracker, analyse_start_time, routes,
- track_break_time=timedelta(minutes=30),
- track_break_dist=10000,
- find_closest_cache=None,
- processing_lock=None,
- ):
- self = cls('analysed.{}'.format(org_tracker.name))
+ async def start(
+ cls,
+ org_tracker,
+ analyse_start_time,
+ routes,
+ track_break_time=timedelta(minutes=30),
+ track_break_dist=10000,
+ find_closest_cache=None,
+ processing_lock=None,
+ ):
+ self = cls("analysed.{}".format(org_tracker.name))
self.org_tracker = org_tracker
self.analyse_start_time = analyse_start_time
self.routes = routes
@@ -65,14 +60,16 @@ async def start(cls, org_tracker, analyse_start_time, routes,
self.completed = asyncio.ensure_future(self._completed())
- self.off_route_tracker = Tracker(f'offroute.{self.name}', completed=self.completed)
- self.pre_post_tracker = Tracker(f'prepost.{self.name}', completed=self.completed)
- self.not_pre_post_observable = Observable(f'{self.name}.not_pre_post')
+ self.off_route_tracker = Tracker(f"offroute.{self.name}", completed=self.completed)
+ self.pre_post_tracker = Tracker(f"prepost.{self.name}", completed=self.completed)
+ self.not_pre_post_observable = Observable(f"{self.name}.not_pre_post")
self.reset()
self.do_est_finish_fut = None
- self.process_initial_points_fut = asyncio.ensure_future(self.on_new_points(self.org_tracker, self.org_tracker.points))
+ self.process_initial_points_fut = asyncio.ensure_future(
+ self.on_new_points(self.org_tracker, self.org_tracker.points)
+ )
self.process_initial_points_fut.add_done_callback(general_fut_done_callback)
self.org_tracker.new_points_observable.subscribe(self.on_new_points)
self.org_tracker.reset_points_observable.subscribe(self.on_reset_points)
@@ -126,8 +123,7 @@ async def on_reset_points(self, tracker):
self.reset()
async def on_new_points(self, tracker, new_points):
- self.logger.debug(
- 'analyse_tracker_new_points ({} points)'.format(len(new_points)))
+ self.logger.debug("analyse_tracker_new_points ({} points)".format(len(new_points)))
if self.do_est_finish_fut:
cancel_and_wait_task(self.do_est_finish_fut)
@@ -154,31 +150,42 @@ async def submit_points():
log_time = datetime.now()
log_i = 0
- last_route_point = self.routes[0]['points'][-1] if self.routes else None
+ last_route_point = self.routes[0]["points"][-1] if self.routes else None
last_point_i = len(new_points) - 1
did_slow_log = False
for i, point in enumerate(new_points):
point = copy.deepcopy(point)
- self.pre_post = pre_post = self.finished or (self.analyse_start_time and self.analyse_start_time > point['time'])
- if 'position' in point:
- point_point = Point(*point['position'][:2])
+ self.pre_post = pre_post = self.finished or (
+ self.analyse_start_time and self.analyse_start_time > point["time"]
+ )
+ if "position" in point:
+ point_point = Point(*point["position"][:2])
if not pre_post:
if self.prev_route_dist_time:
- time_from_prev_route_dist = point['time'] - self.prev_route_dist_time
- max_travel_dist = 300000 * max(time_from_prev_route_dist.total_seconds(), 20) / 3600
+ time_from_prev_route_dist = point["time"] - self.prev_route_dist_time
+ max_travel_dist = (
+ 300000 * max(time_from_prev_route_dist.total_seconds(), 20) / 3600
+ )
elif self.analyse_start_time:
- time_from_start = point['time'] - self.analyse_start_time
- max_travel_dist = 300000 * max(time_from_start.total_seconds(), 20) / 3600
+ time_from_start = point["time"] - self.analyse_start_time
+ max_travel_dist = (
+ 300000 * max(time_from_start.total_seconds(), 20) / 3600
+ )
else:
max_travel_dist = None
closest = self.find_closest(
- self.routes, point_point, 5000,
+ self.routes,
+ point_point,
+ 5000,
self.prev_closest.route_i if self.prev_closest else None,
- 250, self.prev_route_dist or 0, max_travel_dist)
+ 250,
+ self.prev_route_dist or 0,
+ max_travel_dist,
+ )
if closest and closest.dist > 100000:
closest = None
else:
@@ -187,21 +194,31 @@ async def submit_points():
if closest:
on_route = closest.dist < 250
route_dist = route_distance(closest.route, closest)
- point['dist_route'] = round(route_dist)
- self.going_forward = (not self.prev_route_dist) or route_dist > self.prev_route_dist
-
- if on_route and 'elevation' in closest.route:
- point['route_elevation'] = round(route_elevation(closest.route, route_dist))
- if len(point['position']) > 2 and abs(point['route_elevation'] - point['position'][2]) > 500:
- self.logger.debug('Removing inaccurate elevation.')
- point['position'] = point['position'][:2]
-
- if closest.route_i == 0 and abs(route_dist - last_route_point.distance) < 100:
+ point["dist_route"] = round(route_dist)
+ self.going_forward = (
+ not self.prev_route_dist
+ ) or route_dist > self.prev_route_dist
+
+ if on_route and "elevation" in closest.route:
+ point["route_elevation"] = round(
+ route_elevation(closest.route, route_dist)
+ )
+ if (
+ len(point["position"]) > 2
+ and abs(point["route_elevation"] - point["position"][2]) > 500
+ ):
+ self.logger.debug("Removing inaccurate elevation.")
+ point["position"] = point["position"][:2]
+
+ if (
+ closest.route_i == 0
+ and abs(route_dist - last_route_point.distance) < 100
+ ):
# This is for when we get a point at the finish.
- self.logger.debug('Finished')
+ self.logger.debug("Finished")
self.set_finished()
- point['finished_time'] = point['time']
- point['rider_status'] = 'Finished'
+ point["finished_time"] = point["time"]
+ point["rider_status"] = "Finished"
else:
self.going_forward = None
on_route = False
@@ -210,21 +227,30 @@ async def submit_points():
time_from_last = None
if self.prev_point_with_position:
prev_point = self.prev_point_with_position
- point['time_from_last'] = time_from_last = point['time'] - prev_point['time']
- if 'server_time' in point and 'server_time' in prev_point:
- point['server_time_from_last'] = point['server_time'] - prev_point['server_time']
+ point["time_from_last"] = time_from_last = (
+ point["time"] - prev_point["time"]
+ )
+ if "server_time" in point and "server_time" in prev_point:
+ point["server_time_from_last"] = (
+ point["server_time"] - prev_point["server_time"]
+ )
time = None
dist_from_last = None
- if on_route and self.prev_on_route and closest.route_i == self.prev_closest.route_i:
+ if (
+ on_route
+ and self.prev_on_route
+ and closest.route_i == self.prev_closest.route_i
+ ):
dist_from_last = abs(
- route_distance_no_adjust(closest.route, closest) -
- route_distance_no_adjust(self.prev_closest.route, self.prev_closest))
+ route_distance_no_adjust(closest.route, closest)
+ - route_distance_no_adjust(self.prev_closest.route, self.prev_closest)
+ )
time = time_from_last
elif on_route and not self.total_dist:
# Assume the last point was at the start of the route, and at the start of the event.
- time = point['time'] - self.analyse_start_time
+ time = point["time"] - self.analyse_start_time
dist_from_last = route_dist
elif self.prev_point_with_position:
# as the crow flys distance
@@ -234,15 +260,15 @@ async def submit_points():
if dist_from_last:
if not pre_post:
self.total_dist += dist_from_last
- point['dist'] = round(self.total_dist)
- point['dist_from_last'] = round(dist_from_last)
+ point["dist"] = round(self.total_dist)
+ point["dist_from_last"] = round(dist_from_last)
speed_from_last = None
if time and dist_from_last:
seconds = time.total_seconds()
if seconds != 0:
speed_from_last = dist_from_last / seconds * 3.6
- point['speed_from_last'] = round(speed_from_last, 1)
+ point["speed_from_last"] = round(speed_from_last, 1)
if time > self.track_break_time and dist_from_last > self.track_break_dist:
if pre_post:
@@ -251,21 +277,53 @@ async def submit_points():
self.current_track_id += 1
self.off_route_track_id += 1
- if not self.finished and i == last_point_i and closest and closest.route_i == 0 and abs(route_dist - last_route_point.distance) < 2000:
+ if (
+ not self.finished
+ and i == last_point_i
+ and closest
+ and closest.route_i == 0
+ and abs(route_dist - last_route_point.distance) < 2000
+ ):
# This is for when they are near to the finish, and the tracker turns off too soon.
- seconds_to_finish = abs(route_dist - last_route_point.distance) / (dist_from_last / seconds)
- est_finish_time = point['time'] + timedelta(seconds=seconds_to_finish)
- self.do_est_finish_fut = asyncio.ensure_future(self.do_est_finish(est_finish_time))
+ seconds_to_finish = abs(route_dist - last_route_point.distance) / (
+ dist_from_last / seconds
+ )
+ est_finish_time = point["time"] + timedelta(seconds=seconds_to_finish)
+ self.do_est_finish_fut = asyncio.ensure_future(
+ self.do_est_finish(est_finish_time)
+ )
if not pre_post:
- if not on_route or (not self.going_forward and speed_from_last and speed_from_last > 3):
+ if not on_route or (
+ not self.going_forward and speed_from_last and speed_from_last > 3
+ ):
# self.logger.info('off_route')
- if not self.is_off_route and self.prev_point_with_position and self.prev_point_with_position['track_id'] == self.current_track_id:
- new_off_route_points.append({'position': self.prev_point_with_position['position'], 'track_id': self.off_route_track_id})
+ if (
+ not self.is_off_route
+ and self.prev_point_with_position
+ and self.prev_point_with_position["track_id"]
+ == self.current_track_id
+ ):
+ new_off_route_points.append(
+ {
+ "position": self.prev_point_with_position["position"],
+ "track_id": self.off_route_track_id,
+ }
+ )
self.is_off_route = True
- new_off_route_points.append({'position': point['position'], 'track_id': self.off_route_track_id})
+ new_off_route_points.append(
+ {
+ "position": point["position"],
+ "track_id": self.off_route_track_id,
+ }
+ )
elif self.is_off_route:
- new_off_route_points.append({'position': point['position'], 'track_id': self.off_route_track_id})
+ new_off_route_points.append(
+ {
+ "position": point["position"],
+ "track_id": self.off_route_track_id,
+ }
+ )
self.is_off_route = False
self.off_route_track_id += 1
self.non_prepost_prev_point_with_position = point
@@ -275,19 +333,21 @@ async def submit_points():
self.prev_closest = closest
if route_dist:
self.prev_route_dist = route_dist
- self.prev_route_dist_time = point['time']
+ self.prev_route_dist_time = point["time"]
else:
self.prev_route_dist_time = None
self.prev_on_route = on_route
- point['track_id'] = self.pre_post_track_id if pre_post else self.current_track_id
+ point["track_id"] = (
+ self.pre_post_track_id if pre_post else self.current_track_id
+ )
- if not pre_post or 'rider_status' in point:
+ if not pre_post or "rider_status" in point:
new_new_points.append(point)
else:
new_pre_post_points.append(point)
- if 'rider_status' in point:
+ if "rider_status" in point:
self.set_finished()
self.pre_post_track_id += 1
@@ -298,8 +358,14 @@ async def submit_points():
if log_time_delta >= 0.1:
await asyncio.sleep(0)
if log_time_delta >= 1 or (is_last_point and did_slow_log):
- self.logger.info('{}/{} ({:.1f}%) points analysed at {:.2f} points/second.'.format(
- i + 1, len(new_points), (i + 1) / len(new_points) * 100, (i - log_i) / log_time_delta))
+ self.logger.info(
+ "{}/{} ({:.1f}%) points analysed at {:.2f} points/second.".format(
+ i + 1,
+ len(new_points),
+ (i + 1) / len(new_points) * 100,
+ (i - log_i) / log_time_delta,
+ )
+ )
log_time = now
log_i = i
did_slow_log = True
@@ -313,9 +379,9 @@ async def do_est_finish(self, time):
await asyncio.sleep(delay)
self.set_finished()
point = {
- 'finished_time': time,
- 'time': time,
- 'rider_status': 'Finished',
+ "finished_time": time,
+ "time": time,
+ "rider_status": "Finished",
}
await self.new_points([point])
@@ -329,38 +395,45 @@ def get_predicted_position(self, time):
closest = self.prev_closest
if (
- not self.pre_post and pp and time - pp['time'] < self.track_break_time and
- pp.get('speed_from_last', 0) > 3 and self.prev_on_route and self.going_forward):
- time_from_last = (time - pp['time']).total_seconds()
- dist_moved_from_last = pp['speed_from_last'] / 3.6 * time_from_last
+ not self.pre_post
+ and pp
+ and time - pp["time"] < self.track_break_time
+ and pp.get("speed_from_last", 0) > 3
+ and self.prev_on_route
+ and self.going_forward
+ ):
+ time_from_last = (time - pp["time"]).total_seconds()
+ dist_moved_from_last = pp["speed_from_last"] / 3.6 * time_from_last
# Predicted to follow route if they are on the route and going forward.
- dist_route = pp['dist_route'] + dist_moved_from_last
- proceeding_route = list(chain(
- (closest.point, ),
- closest.route['points'][closest.point_pair[1].index:],
- ))
+ dist_route = pp["dist_route"] + dist_moved_from_last
+ proceeding_route = list(
+ chain(
+ (closest.point,),
+ closest.route["points"][closest.point_pair[1].index :],
+ )
+ )
# TODO continue main route
point_point = move_along_route(proceeding_route, dist_moved_from_last)
point = {
- 'position': [point_point.lat, point_point.lng],
- 'dist_route': round(dist_route),
- 'time': time
+ "position": [point_point.lat, point_point.lng],
+ "dist_route": round(dist_route),
+ "time": time,
}
- if 'elevation' in closest.route:
- point['route_elevation'] = round(route_elevation(closest.route, dist_route))
+ if "elevation" in closest.route:
+ point["route_elevation"] = round(route_elevation(closest.route, dist_route))
return point
if pp:
point = {
- 'position': pp['position'],
- 'time': pp['time'],
+ "position": pp["position"],
+ "time": pp["time"],
}
- if 'dist_route' in pp:
- point['dist_route'] = pp['dist_route']
- if 'route_elevation' in pp:
- point['route_elevation'] = pp['route_elevation']
+ if "dist_route" in pp:
+ point["dist_route"] = pp["dist_route"]
+ if "route_elevation" in pp:
+ point["route_elevation"] = pp["route_elevation"]
return point
@@ -411,20 +484,34 @@ def get_analyse_routes(org_routes):
def get_analyse_route(org_route):
route = copy.copy(org_route)
- route['points'] = route_points = route_with_distance_and_index(org_route['points'])
- route['point_pairs'] = [get_point_pair_precalc(*point_pair) for point_pair in pairs(route_points)]
+ route["points"] = route_points = route_with_distance_and_index(org_route["points"])
+ route["point_pairs"] = [
+ get_point_pair_precalc(*point_pair) for point_pair in pairs(route_points)
+ ]
- if route.get('simplified_points_indexes'):
- simplified_points = [route_points[i] for i in route['simplified_points_indexes']]
+ if route.get("simplified_points_indexes"):
+ simplified_points = [route_points[i] for i in route["simplified_points_indexes"]]
else:
- logging.info('No pre-calculated simplified_points. Please run process_event_routes on event for faster start up.')
- if not route.get('split_at_dist'):
+ logging.info(
+ "No pre-calculated simplified_points. Please run process_event_routes on event for faster start up."
+ )
+ if not route.get("split_at_dist"):
simplified_points = ramer_douglas_peucker(route_points, 500)
else:
- simplified_points = ramer_douglas_peucker_sections(route_points, 500, route['split_at_dist'], route['split_point_range'])
-
- route['simplfied_point_pairs'] = [get_point_pair_precalc(*point_pair) for point_pair in pairs(simplified_points)]
- logger.debug('Route points: {}, simplified points: {}, distance: {}'.format(len(route_points), len(route['simplfied_point_pairs']), route_points[-1].distance))
+ simplified_points = ramer_douglas_peucker_sections(
+ route_points, 500, route["split_at_dist"], route["split_point_range"]
+ )
+
+ route["simplfied_point_pairs"] = [
+ get_point_pair_precalc(*point_pair) for point_pair in pairs(simplified_points)
+ ]
+ logger.debug(
+ "Route points: {}, simplified points: {}, distance: {}".format(
+ len(route_points),
+ len(route["simplfied_point_pairs"]),
+ route_points[-1].distance,
+ )
+ )
return route
@@ -441,6 +528,7 @@ def get_point(i, point):
point.distance = dist
previous_point = point
return point
+
return [get_point(i, point) for i, point in enumerate(route)]
@@ -457,7 +545,7 @@ def pairs(items):
item1 = item2
-dist_attr_getter = operator.attrgetter('dist')
+dist_attr_getter = operator.attrgetter("dist")
def ramer_douglas_peucker(points, epsilon):
@@ -468,8 +556,8 @@ def ramer_douglas_peucker(points, epsilon):
dmax = 0
if dmax > epsilon:
- r1 = ramer_douglas_peucker(points[:imax + 2], epsilon)
- r2 = ramer_douglas_peucker(points[imax + 1:], epsilon)
+ r1 = ramer_douglas_peucker(points[: imax + 2], epsilon)
+ r2 = ramer_douglas_peucker(points[imax + 1 :], epsilon)
return r1[:-1] + r2
else:
return (points[0], points[-1])
@@ -486,7 +574,9 @@ def ramer_douglas_peucker_sections(points, epsilon, split_at_dist, split_point_r
simplified_close_points = ramer_douglas_peucker(close_points, epsilon)
closest_point = min(simplified_close_points, key=lambda point: abs(dist - point.distance))
closest_index = closest_point.index
- simplified_points_section = ramer_douglas_peucker(points[last_index:closest_index + 1], epsilon)
+ simplified_points_section = ramer_douglas_peucker(
+ points[last_index : closest_index + 1], epsilon
+ )
simplified_points_sections.append(simplified_points_section[:-1])
last_index = closest_index
@@ -494,29 +584,60 @@ def ramer_douglas_peucker_sections(points, epsilon, split_at_dist, split_point_r
return list(chain.from_iterable(simplified_points_sections))
-find_closest_point_pair_routes_result = collections.namedtuple('closest_point_pair_route', ('route_i', 'route', 'point_pair', 'dist', 'point'))
+find_closest_point_pair_routes_result = collections.namedtuple(
+ "closest_point_pair_route", ("route_i", "route", "point_pair", "dist", "point")
+)
-def find_closest_point_pair_routes_cache_key(routes, to_point, min_search_complex_dist, prev_closest_route_i, break_out_dist, prev_dist, max_travel_dist):
- return to_point.lat, to_point.lng, min_search_complex_dist, prev_closest_route_i, break_out_dist, prev_dist, max_travel_dist
+def find_closest_point_pair_routes_cache_key(
+ routes,
+ to_point,
+ min_search_complex_dist,
+ prev_closest_route_i,
+ break_out_dist,
+ prev_dist,
+ max_travel_dist,
+):
+ return (
+ to_point.lat,
+ to_point.lng,
+ min_search_complex_dist,
+ prev_closest_route_i,
+ break_out_dist,
+ prev_dist,
+ max_travel_dist,
+ )
def find_closest_point_pair_routes_pack(result):
if result:
- return result.route_i, result.point_pair[0].index, result.dist, result.point.lat, result.point.lng
+ return (
+ result.route_i,
+ result.point_pair[0].index,
+ result.dist,
+ result.point.lat,
+ result.point.lng,
+ )
def find_closest_point_pair_routes_unpack(routes, packed):
if packed:
route_i, point_pair_index, dist, lat, lng = packed
route = routes[route_i]
- point_pair = route['point_pairs'][point_pair_index]
+ point_pair = route["point_pairs"][point_pair_index]
point = Point(lat, lng)
return find_closest_point_pair_routes_result(route_i, route, point_pair, dist, point)
-def find_closest_point_pair_routes(routes, to_point, min_search_complex_dist, prev_closest_route_i, break_out_dist, prev_dist, max_travel_dist):
-
+def find_closest_point_pair_routes(
+ routes,
+ to_point,
+ min_search_complex_dist,
+ prev_closest_route_i,
+ break_out_dist,
+ prev_dist,
+ max_travel_dist,
+):
# print(to_point)
# import math
# debug = math.isclose(to_point.lat, -28.281551, rel_tol=0.000001) and math.isclose(to_point.lng, 28.93720, rel_tol=0.000001)
@@ -532,9 +653,9 @@ def find_closest_point_pair_routes(routes, to_point, min_search_complex_dist, pr
return find_closest_point_pair_routes_result(0, routes[0], *raw_result)
elif len_routes > 1:
results = []
- special_routes = (0, )
+ special_routes = (0,)
if prev_closest_route_i:
- special_routes += (prev_closest_route_i, )
+ special_routes += (prev_closest_route_i,)
for route_i in reversed(special_routes):
route = routes[route_i]
@@ -547,46 +668,68 @@ def find_closest_point_pair_routes(routes, to_point, min_search_complex_dist, pr
for route_i, route in enumerate(routes):
if route_i not in special_routes:
- raw_result = find_closest_point_pair_route(route, to_point, prev_dist, max_travel_dist)
+ raw_result = find_closest_point_pair_route(
+ route, to_point, prev_dist, max_travel_dist
+ )
if raw_result:
- results.append(find_closest_point_pair_routes_result(route_i, route, *raw_result))
+ results.append(
+ find_closest_point_pair_routes_result(route_i, route, *raw_result)
+ )
if results:
return min(results, key=dist_attr_getter)
-find_closest_point_pair_result = collections.namedtuple('closest_point_pair', ('point_pair', 'dist', 'point'))
+find_closest_point_pair_result = collections.namedtuple(
+ "closest_point_pair", ("point_pair", "dist", "point")
+)
def find_closest_point_pair_route(route, to_point, prev_dist, max_travel_dist):
if max_travel_dist:
min_route_dist = prev_dist - max_travel_dist
max_route_dist = prev_dist + max_travel_dist
- if route['main']:
+ if route["main"]:
get_point_distance = lambda point: point.distance
else:
- get_point_distance = lambda point: point.distance * route['dist_factor'] + route['start_distance']
-
- test_point_pair = lambda point_pair: get_point_distance(point_pair[1]) > min_route_dist and get_point_distance(point_pair[0]) <= max_route_dist
+ get_point_distance = (
+ lambda point: point.distance * route["dist_factor"] + route["start_distance"]
+ )
+
+ test_point_pair = (
+ lambda point_pair: get_point_distance(point_pair[1]) > min_route_dist
+ and get_point_distance(point_pair[0]) <= max_route_dist
+ )
else:
test_point_pair = lambda point_pair: True
simplified_c_points = (
- find_closest_point_pair_result(point_pair[:2], *find_c_point_from_precalc(to_point, *point_pair))
- for point_pair in route['simplfied_point_pairs']
+ find_closest_point_pair_result(
+ point_pair[:2], *find_c_point_from_precalc(to_point, *point_pair)
+ )
+ for point_pair in route["simplfied_point_pairs"]
if test_point_pair(point_pair)
)
simplified_c_points_sorted = sorted(simplified_c_points, key=dist_attr_getter)
simplified_c_points_top = take(4, simplified_c_points_sorted)
- simplified_c_points_filtered = filter(lambda closest: closest.dist < 100000, simplified_c_points_top)
+ simplified_c_points_filtered = filter(
+ lambda closest: closest.dist < 100000, simplified_c_points_top
+ )
- route_point_pairs = route['point_pairs']
- route_point_pairs_filtered = chain.from_iterable((
- route_point_pairs[simplified_c_point.point_pair[0].index: simplified_c_point.point_pair[1].index + 1]
- for simplified_c_point in simplified_c_points_filtered))
+ route_point_pairs = route["point_pairs"]
+ route_point_pairs_filtered = chain.from_iterable(
+ (
+ route_point_pairs[
+ simplified_c_point.point_pair[0].index : simplified_c_point.point_pair[1].index + 1
+ ]
+ for simplified_c_point in simplified_c_points_filtered
+ )
+ )
with_c_points = (
- find_closest_point_pair_result(point_pair[:2], *find_c_point_from_precalc(to_point, *point_pair))
+ find_closest_point_pair_result(
+ point_pair[:2], *find_c_point_from_precalc(to_point, *point_pair)
+ )
for point_pair in route_point_pairs_filtered
if test_point_pair(point_pair)
)
@@ -599,8 +742,9 @@ def find_closest_point_pair_route(route, to_point, prev_dist, max_travel_dist):
head, with_c_points = spy(with_c_points)
if head:
- circular_range = route.get('circular_range')
+ circular_range = route.get("circular_range")
if prev_dist is not None and circular_range:
+
def min_key(closest):
if closest.dist > 100000:
# Short cut to avoid unnecessary route_distance calls
@@ -617,6 +761,7 @@ def min_key(closest):
# if debug:
# print(rank, move_distance, move_distance_penalty, closest, )
return rank
+
else:
min_key = dist_attr_getter
@@ -627,25 +772,27 @@ def min_key(closest):
def route_distance(route, closest):
prev_route_point = closest.point_pair[0]
- if route['main']:
+ if route["main"]:
return round(prev_route_point.distance + distance(prev_route_point, closest.point))
else:
alt_route_dist = prev_route_point.distance + distance(prev_route_point, closest.point)
- return round(alt_route_dist * route['dist_factor'] + route['start_distance'])
+ return round(alt_route_dist * route["dist_factor"] + route["start_distance"])
def route_distance_no_adjust(route, closest):
prev_route_point = closest.point_pair[0]
- start_distance = 0 if route['main'] else route['start_distance']
- return round(prev_route_point.distance + distance(prev_route_point, closest.point) + start_distance)
+ start_distance = 0 if route["main"] else route["start_distance"]
+ return round(
+ prev_route_point.distance + distance(prev_route_point, closest.point) + start_distance
+ )
def route_elevation(route, route_dist):
- elevation = route['elevation']
- if route['main']:
+ elevation = route["elevation"]
+ if route["main"]:
dist_on_route = route_dist
else:
- dist_on_route = (route_dist - route['start_distance']) / route['dist_factor']
+ dist_on_route = (route_dist - route["start_distance"]) / route["dist_factor"]
point2_i = bisect.bisect(KeyifyList(elevation, itemgetter(3)), dist_on_route)
if point2_i == len(elevation):
point2_i -= 1
@@ -657,7 +804,7 @@ def route_elevation(route, route_dist):
return ((point2[2] - point1[2]) * dist_factor) + point2[2]
-find_c_point_result = collections.namedtuple('c_point', ('dist', 'point'))
+find_c_point_result = collections.namedtuple("c_point", ("dist", "point"))
def find_c_point(to_point, point1, point2):
@@ -682,7 +829,7 @@ def find_c_point_from_precalc(to_point, point1, point2, c12, p1h, p2h, dp1p2):
raise
sutable_c = None
for co in (c, 0 - c):
- co_rs = co.reshape((3, ))
+ co_rs = co.reshape((3,))
dp1co = arccos(arccos_limit(dot(p1h, co_rs)))
dp2co = arccos(arccos_limit(dot(p2h, co_rs)))
if abs(dp1co + dp2co - dp1p2) < 0.000001:
@@ -694,7 +841,9 @@ def find_c_point_from_precalc(to_point, point1, point2, c12, p1h, p2h, dp1p2):
c_point = Point(lat=rad2deg(c_point_lat[0]), lng=rad2deg(c_point_lng[0]))
c_dist = distance(to_point, c_point)
else:
- c_dist, c_point = min(((distance(to_point, p), p) for p in (point1, point2)), key=itemgetter(0))
+ c_dist, c_point = min(
+ ((distance(to_point, p), p) for p in (point1, point2)), key=itemgetter(0)
+ )
return find_c_point_result(c_dist, c_point)
@@ -703,8 +852,8 @@ def get_point_pair_precalc(point1, point2):
p1 = point1.nv
p2 = point2.nv
c12 = cross(p1, p2, axis=0)
- p1h = p1.reshape((3, ))
- p2h = p2.reshape((3, ))
+ p1h = p1.reshape((3,))
+ p2h = p2.reshape((3,))
dp1p2 = arccos(dot(p1h, p2h))
return point1, point2, c12, p1h, p2h, dp1p2
@@ -720,7 +869,10 @@ def get_equal_spaced_points(points, dist_between_points, start_dist=0, round_dig
while point_dist_remaining > dist_between_points:
point_dist_remaining -= dist_between_points
cum_dist += dist_between_points
- new_point_nv = interpolate((last_point.nv, point.nv), (point_distance - point_dist_remaining) / point_distance)
+ new_point_nv = interpolate(
+ (last_point.nv, point.nv),
+ (point_distance - point_dist_remaining) / point_distance,
+ )
new_point = Point.from_nv(new_point_nv, round_digits=round_digits)
yield (new_point, cum_dist)
dist_from_last_step = point_dist_remaining
@@ -731,7 +883,11 @@ def get_equal_spaced_points(points, dist_between_points, start_dist=0, round_dig
def move_along_route(route, dist):
for i, (point1, point2) in enumerate(pairs(route)):
- dist_between = point2.distance - point1.distance if isinstance(point1, IndexedPoint) and isinstance(point2, IndexedPoint) else distance(point1, point2)
+ dist_between = (
+ point2.distance - point1.distance
+ if isinstance(point1, IndexedPoint) and isinstance(point2, IndexedPoint)
+ else distance(point1, point2)
+ )
if dist > dist_between:
dist -= dist_between
else:
diff --git a/trackers/auth.py b/trackers/auth.py
index c6f5394..cea5e55 100644
--- a/trackers/auth.py
+++ b/trackers/auth.py
@@ -10,46 +10,48 @@
from cryptography import fernet
from htmlwrite import Markup, Tag, Writer
-
logger = logging.getLogger(__name__)
def gen_key():
- print(fernet.Fernet.generate_key().decode('ascii'))
+ print(fernet.Fernet.generate_key().decode("ascii"))
async def config_aio_app(app, settings):
-
- app['oauth_providers'] = settings['oauth_providers']
- app['oauth_base_url'] = settings['app_url']
- app['oauth_providers_by_name'] = {provider['name']: provider for provider in settings['oauth_providers']}
- app['authorization'] = settings.get('authorization', True)
+ app["oauth_providers"] = settings["oauth_providers"]
+ app["oauth_base_url"] = settings["app_url"]
+ app["oauth_providers_by_name"] = {
+ provider["name"]: provider for provider in settings["oauth_providers"]
+ }
+ app["authorization"] = settings.get("authorization", True)
max_age = int(timedelta(days=30).total_seconds())
- storage = aiohttp_session.cookie_storage.EncryptedCookieStorage(settings['aiosession_encryption_key'], max_age=max_age)
+ storage = aiohttp_session.cookie_storage.EncryptedCookieStorage(
+ settings["aiosession_encryption_key"], max_age=max_age
+ )
aiohttp_session.setup(app, storage)
- app.router.add_route('GET', '/oauth/{provider}', oauth, name='oauth_login')
- app.router.add_route('GET', '/logout', logout, name='logout')
- app.router.add_route('GET', '/get_identity', get_identity, name='get_identity')
+ app.router.add_route("GET", "/oauth/{provider}", oauth, name="oauth_login")
+ app.router.add_route("GET", "/logout", logout, name="logout")
+ app.router.add_route("GET", "/get_identity", get_identity, name="get_identity")
async def oauth(request):
- providers_by_name = request.app['oauth_providers_by_name']
- provider = request.match_info.get('provider')
+ providers_by_name = request.app["oauth_providers_by_name"]
+ provider = request.match_info.get("provider")
if provider not in providers_by_name:
- raise web.HTTPNotFound(reason='Unknown provider')
+ raise web.HTTPNotFound(reason="Unknown provider")
# Create OAuth1/2 client
Client = aioauth_client.ClientRegistry.clients[provider]
- params = providers_by_name[provider]['init']
+ params = providers_by_name[provider]["init"]
client = Client(**params)
- client.params['redirect_uri'] = '{}{}'.format(request.app['oauth_base_url'], request.path)
+ client.params["redirect_uri"] = "{}{}".format(request.app["oauth_base_url"], request.path)
if client.shared_key not in request.query:
# Redirect client to provider
- if request.query.get('return_to'):
- client.params['state'] = request.query['return_to']
+ if request.query.get("return_to"):
+ client.params["state"] = request.query["return_to"]
return web.HTTPFound(client.get_authorize_url())
await client.get_access_token(request.query[client.shared_key])
@@ -57,23 +59,23 @@ async def oauth(request):
session = await aiohttp_session.get_session(request)
- session['identity'] = dict(client.user_parse(user_data))
+ session["identity"] = dict(client.user_parse(user_data))
- return_to = request.query.get('state')
+ return_to = request.query.get("state")
if not return_to:
- return_to = '/'
+ return_to = "/"
return web.HTTPFound(return_to)
async def logout(request):
session = await aiohttp_session.get_session(request)
- session['identity'] = None
- return web.HTTPFound(request.query.get('return_to', '/'))
+ session["identity"] = None
+ return web.HTTPFound(request.query.get("return_to", "/"))
async def get_identity(request):
session = await aiohttp_session.get_session(request)
- return session.get('identity')
+ return session.get("identity")
async def get_git_author(request):
@@ -92,58 +94,102 @@ async def show_identity(request, writer, identity=sentinel):
w = writer.write
if identity:
- with c(Tag('div')):
- if identity.get('picture'):
- w(Tag('img', src=identity['picture'], s_height="4em", s_margin_right="8px"))
- with c(Tag('div', s_display='inline-block')):
- w('You are logged in as:')
- w(Tag('br'))
+ with c(Tag("div")):
+ if identity.get("picture"):
+ w(
+ Tag(
+ "img",
+ src=identity["picture"],
+ s_height="4em",
+ s_margin_right="8px",
+ )
+ )
+ with c(Tag("div", s_display="inline-block")):
+ w("You are logged in as:")
+ w(Tag("br"))
w(f"{identity['first_name']} <{identity['email']}>")
- w(Tag('br'))
-
- w(Tag('a', class_="waves-effect waves-light btn",
- href=request.app.router['logout'].url_for().with_query(return_to=request.path)),
- 'Logout')
+ w(Tag("br"))
+
+ w(
+ Tag(
+ "a",
+ class_="waves-effect waves-light btn",
+ href=request.app.router["logout"].url_for().with_query(return_to=request.path),
+ ),
+ "Logout",
+ )
else:
- with c(Tag('div')):
- w('Not logged in. Login with: ')
-
- for provider in request.app['oauth_providers']:
- w(Tag('a', class_="waves-effect waves-light btn",
- href=request.app.router['oauth_login'].url_for(provider=provider['name']).with_query(return_to=request.path)),
- provider['display_name'])
+ with c(Tag("div")):
+ w("Not logged in. Login with: ")
+
+ for provider in request.app["oauth_providers"]:
+ w(
+ Tag(
+ "a",
+ class_="waves-effect waves-light btn",
+ href=request.app.router["oauth_login"]
+ .url_for(provider=provider["name"])
+ .with_query(return_to=request.path),
+ ),
+ provider["display_name"],
+ )
async def ensure_authorized_handler(request, allowed_principals, handler):
- if not request.app['authorization']:
+ if not request.app["authorization"]:
return await handler(request)
else:
identity = await get_identity(request)
- if identity and identity['email'] in allowed_principals:
+ if identity and identity["email"] in allowed_principals:
return await handler(request)
else:
body = io.StringIO()
writer = Writer(body)
w = writer.w
c = writer.c
- error = 'Not Authorised.' if identity else 'Not Authenticated'
- w(Markup(''))
- with c(Tag('html')):
- with c(Tag('head')):
- w(Tag('title'), error)
- w(Tag('meta', name="viewport", content="initial-scale=1.0, user-scalable=no"))
- w(Tag('link', rel="stylesheet",
- href="/static/contrib/materialize.min.css"))
- with c(Tag('body', s_padding="24px", s_width="100%", )):
- with c(Tag('div', s_margin="auto", s_max_width="800px", )):
- w(Tag('h1'), error)
+ error = "Not Authorised." if identity else "Not Authenticated"
+ w(Markup(""))
+ with c(Tag("html")):
+ with c(Tag("head")):
+ w(Tag("title"), error)
+ w(
+ Tag(
+ "meta",
+ name="viewport",
+ content="initial-scale=1.0, user-scalable=no",
+ )
+ )
+ w(
+ Tag(
+ "link",
+ rel="stylesheet",
+ href="/static/contrib/materialize.min.css",
+ )
+ )
+ with c(
+ Tag(
+ "body",
+ s_padding="24px",
+ s_width="100%",
+ )
+ ):
+ with c(
+ Tag(
+ "div",
+ s_margin="auto",
+ s_max_width="800px",
+ )
+ ):
+ w(Tag("h1"), error)
await show_identity(request, writer)
- return web.Response(body=body.getvalue(), headers={'Content-Type': 'text/html; charset=utf-8'},
- status=403 if identity else 401)
+ return web.Response(
+ body=body.getvalue(),
+ headers={"Content-Type": "text/html; charset=utf-8"},
+ status=403 if identity else 401,
+ )
def ensure_authorized(handler, allowed_principals):
-
async def ensure_authorized_inner(request):
return await ensure_authorized_handler(request, allowed_principals, handler)
@@ -151,10 +197,11 @@ async def ensure_authorized_inner(request):
def ensure_authorized_event(handler):
-
async def ensure_authorized_inner(request, event):
allowed_principals = event.admin_allowed_principals
- logger.debug(f'allowed_principals: {allowed_principals}')
- return await ensure_authorized_handler(request, allowed_principals, functools.partial(handler, event=event))
+ logger.debug(f"allowed_principals: {allowed_principals}")
+ return await ensure_authorized_handler(
+ request, allowed_principals, functools.partial(handler, event=event)
+ )
return ensure_authorized_inner
diff --git a/trackers/base.py b/trackers/base.py
index fbacb28..9f9705d 100644
--- a/trackers/base.py
+++ b/trackers/base.py
@@ -14,14 +14,17 @@
class Tracker(object):
-
def __init__(self, name, completed=None, new_points_callbacks=(), reset_points_callbacks=()):
self.name = name
self.points = []
self.status = None
- self.logger = logging.getLogger('trackers.{}'.format(name))
- self.new_points_observable = Observable(f'{self.name}.new_points', callbacks=new_points_callbacks)
- self.reset_points_observable = Observable(f'{self.name}.reset_points', callbacks=reset_points_callbacks)
+ self.logger = logging.getLogger("trackers.{}".format(name))
+ self.new_points_observable = Observable(
+ f"{self.name}.new_points", callbacks=new_points_callbacks
+ )
+ self.reset_points_observable = Observable(
+ f"{self.name}.reset_points", callbacks=reset_points_callbacks
+ )
self.callback_tasks = []
if completed is None:
@@ -32,7 +35,7 @@ def __init__(self, name, completed=None, new_points_callbacks=(), reset_points_c
self.finished = False
def __repr__(self):
- return f'<{type(self).__name__}({self.name})>'
+ return f"<{type(self).__name__}({self.name})>"
async def new_points(self, new_points):
self.points.extend(new_points)
@@ -58,13 +61,12 @@ def set_finished(self):
class Observable(object):
-
- def __init__(self, name, callbacks=(), error_msg='Error calling callback: '):
+ def __init__(self, name, callbacks=(), error_msg="Error calling callback: "):
self.name = name
self.callbacks = []
self.callbacks.extend(callbacks)
self.error_msg = error_msg
- self.logger = logging.getLogger(f'observable.{name}')
+ self.logger = logging.getLogger(f"observable.{name}")
def subscribe(self, callback):
self.callbacks.append(callback)
@@ -74,14 +76,14 @@ def unsubscribe(self, callback):
async def __call__(self, *args, **kwargs):
if self.logger.isEnabledFor(logging.DEBUG):
- self.logger.debug(f'Calling {self.callbacks}(*{args}, **{kwargs})'[:1000])
+ self.logger.debug(f"Calling {self.callbacks}(*{args}, **{kwargs})"[:1000])
for callback in self.callbacks:
try:
await callback(*args, **kwargs)
except asyncio.CancelledError:
raise
except Exception:
- self.logger.exception(f'{self.error_msg} ({callback, args, kwargs})')
+ self.logger.exception(f"{self.error_msg} ({callback, args, kwargs})")
async def cancel_and_wait_task(task):
@@ -98,7 +100,7 @@ def general_fut_done_callback(fut):
except asyncio.CancelledError:
pass
except Exception:
- logging.exception('')
+ logging.exception("")
def run_forget_task(coro):
@@ -119,14 +121,13 @@ def list_register(list, item, on_empty=None, yield_item=None):
def print_tracker(tracker):
-
async def print_callback(callback, source, data):
- print('{} {}: \n{}'.format(source.name, callback, pprint.pformat(data)))
+ print("{} {}: \n{}".format(source.name, callback, pprint.pformat(data)))
- tracker.new_points_observable.subscribe(functools.partial(print_callback, 'new_points'))
+ tracker.new_points_observable.subscribe(functools.partial(print_callback, "new_points"))
for point in tracker.points:
- print('{} {}: \n{}'.format(tracker.name, None, pprint.pformat(point)))
+ print("{} {}: \n{}".format(tracker.name, None, pprint.pformat(point)))
def get_blocked_list(source, existing, smallest_block_len=8, entire_block=False):
@@ -140,47 +141,65 @@ def get_blocked_list(source, existing, smallest_block_len=8, entire_block=False)
while block_i + block_len < source_len:
end_index = block_i + block_len - 1
- blocks.append({'start_index': block_i, 'end_index': end_index, 'end_hash': source[end_index]['hash']})
+ blocks.append(
+ {
+ "start_index": block_i,
+ "end_index": end_index,
+ "end_hash": source[end_index]["hash"],
+ }
+ )
block_i += block_len
partial_block = list(source[block_i:])
else:
if source:
- blocks = [{'start_index': 0, 'end_index': source[-1]['index'], 'end_hash': source[-1]['hash'], }]
+ blocks = [
+ {
+ "start_index": 0,
+ "end_index": source[-1]["index"],
+ "end_hash": source[-1]["hash"],
+ }
+ ]
else:
blocks = []
partial_block = []
- full = {'blocks': blocks, 'partial_block': partial_block}
+ full = {"blocks": blocks, "partial_block": partial_block}
- if existing.get('blocks') != blocks:
+ if existing.get("blocks") != blocks:
update = full
else:
- existing_partial_block = existing.get('partial_block', ())
+ existing_partial_block = existing.get("partial_block", ())
if len(existing_partial_block) > len(partial_block):
- update = {'partial_block': partial_block}
+ update = {"partial_block": partial_block}
else:
- for existing_item, item in zip(existing_partial_block, partial_block[:len(existing_partial_block)]):
- if (existing_item['hash'], existing_item['hash']) != (item['hash'], item['hash']):
- update = {'partial_block': partial_block}
+ for existing_item, item in zip(
+ existing_partial_block, partial_block[: len(existing_partial_block)]
+ ):
+ if (existing_item["hash"], existing_item["hash"]) != (
+ item["hash"],
+ item["hash"],
+ ):
+ update = {"partial_block": partial_block}
break
else:
- add_block = partial_block[len(existing_partial_block):]
+ add_block = partial_block[len(existing_partial_block) :]
if add_block:
- update = {'add_block': add_block}
+ update = {"add_block": add_block}
else:
update = {}
return full, update
class BlockedList(object):
-
def __init__(self, source_name, get_source, new_update_callbacks=(), **kwargs):
self.get_source = get_source
self.kwargs = kwargs
self.full, _ = get_blocked_list(get_source(), {}, **self.kwargs)
self.last = self.full
- self.new_update_observable = Observable(f'{source_name}.blocked_list_new_update', callbacks=new_update_callbacks)
+ self.new_update_observable = Observable(
+ f"{source_name}.blocked_list_new_update", callbacks=new_update_callbacks
+ )
@staticmethod
def from_tracker(tracker, **kwargs):
@@ -206,18 +225,18 @@ def get_update_from_last(self):
# TODO create async version that uses io executor
@contextlib.contextmanager
def stream_store(path: Path, logger: logging.Logger):
- logging.debug('Reading data')
+ logging.debug("Reading data")
if path.exists():
- with path.open('rb') as f:
+ with path.open("rb") as f:
unpacker = msgpack.Unpacker(f, raw=False, timestamp=3)
data = list(chain.from_iterable(unpacker))
- logging.info('Data loaded: {} items'.format(len(data)))
+ logging.info("Data loaded: {} items".format(len(data)))
else:
path.parent.mkdir(parents=True, exist_ok=True)
data = []
- logging.info('Data file did not exist. Starting blank.')
+ logging.info("Data file did not exist. Starting blank.")
- with path.open('ab', 0) as f:
+ with path.open("ab", 0) as f:
packer = msgpack.Packer(datetime=True)
def write_items(items):
@@ -230,6 +249,7 @@ def write_items(items):
# From https://stackoverflow.com/a/46255794/72911
+
class RateLimitingSemaphore:
def __init__(self, qps_limit, loop=None):
self.loop = loop or asyncio.get_event_loop()
@@ -250,7 +270,7 @@ async def __aenter__(self):
cur_rate = len(self.call_times) / (self.loop.time() - self.call_times[0])
if cur_rate < self.qps_limit:
break
- interval = 1. / self.qps_limit
+ interval = 1.0 / self.qps_limit
elapsed_time = self.loop.time() - self.call_times[-1]
await asyncio.sleep(self.queued_calls * interval - elapsed_time)
self.queued_calls -= 1
diff --git a/trackers/bin_utils.py b/trackers/bin_utils.py
index 5b1a722..4bfe12e 100644
--- a/trackers/bin_utils.py
+++ b/trackers/bin_utils.py
@@ -15,22 +15,22 @@
import dulwich.repo
import polyline
import yaml
+from dulwich_tree import TreeReader, TreeWriter
from more_itertools import chunked, interleave_longest
import trackers.events
import trackers.general
import trackers.modules
from trackers.analyse import (
+ IndexedPoint,
distance,
find_closest_point_pair_route,
get_analyse_route,
get_equal_spaced_points,
- IndexedPoint,
ramer_douglas_peucker,
ramer_douglas_peucker_sections,
route_with_distance_and_index,
)
-from trackers.dulwich_helpers import TreeReader, TreeWriter
defaults_yaml = f"""
data_path: {relpath(join(__file__, '../../data'))}
@@ -68,13 +68,20 @@ class : logging.StreamHandler
def get_base_argparser(*args, **kwargs):
parser = argparse.ArgumentParser()
- parser.add_argument('settings_file', action='store', nargs='?',
- default=None,
- help='File to load settings from.')
- parser.add_argument('--google-api-key', action='store',
- help='Google api key. ')
- parser.add_argument("--debug", nargs='*', metavar='logger',
- help="Which loggers to set to debug. Arg on it's own will set debug for all loggers.")
+ parser.add_argument(
+ "settings_file",
+ action="store",
+ nargs="?",
+ default=None,
+ help="File to load settings from.",
+ )
+ parser.add_argument("--google-api-key", action="store", help="Google api key. ")
+ parser.add_argument(
+ "--debug",
+ nargs="*",
+ metavar="logger",
+ help="Which loggers to set to debug. Arg on it's own will set debug for all loggers.",
+ )
return parser
@@ -88,9 +95,9 @@ def get_combined_settings(specific_defaults_yaml=None, args=None):
specific_defaults = yaml.safe_load(specific_defaults_yaml)
settings.update(specific_defaults)
- if 'TRACKERS_SETTINGS_FILE' in os.environ:
+ if "TRACKERS_SETTINGS_FILE" in os.environ:
try:
- with open(os.environ['TRACKERS_SETTINGS_FILE']) as f:
+ with open(os.environ["TRACKERS_SETTINGS_FILE"]) as f:
settings_from_file = yaml.safe_load(f)
except FileNotFoundError:
settings_from_file = {}
@@ -100,10 +107,15 @@ def get_combined_settings(specific_defaults_yaml=None, args=None):
settings_file = args.settings_file
if settings_file is None:
paths = (
- 'settings.yaml',
- '../settings.yaml',
- '/etc/trackers.yaml',
- relpath(join(__file__, '../../settings.yaml', ))
+ "settings.yaml",
+ "../settings.yaml",
+ "/etc/trackers.yaml",
+ relpath(
+ join(
+ __file__,
+ "../../settings.yaml",
+ )
+ ),
)
for path in paths:
@@ -112,7 +124,7 @@ def get_combined_settings(specific_defaults_yaml=None, args=None):
break
if settings_file is None:
- logging.warn(f'Settings file not specified, and no defaults not exist ({paths}).')
+ logging.warn(f"Settings file not specified, and no defaults not exist ({paths}).")
else:
try:
with open(settings_file) as f:
@@ -121,20 +133,20 @@ def get_combined_settings(specific_defaults_yaml=None, args=None):
settings_from_file = {}
settings.update(settings_from_file)
- logging.config.dictConfig(settings['logging'])
+ logging.config.dictConfig(settings["logging"])
if args:
if args.debug == []:
- args.debug = ['trackers', 'web_app']
-
+ args.debug = ["trackers", "web_app"]
+
if args.debug:
for logger_name in args.debug:
logging.getLogger(logger_name).setLevel(logging.DEBUG)
- settings['debug'] = args and args.debug
+ settings["debug"] = args and args.debug
- if sys.stdout.isatty() and settings['logging'] != defaults['logging']:
+ if sys.stdout.isatty() and settings["logging"] != defaults["logging"]:
# Reapply the default logging settings
- logging.config.dictConfig(defaults['logging'])
+ logging.config.dictConfig(defaults["logging"])
# settings_dump = yaml.dump(settings)
# logging.getLogger('trackers').info('Combined Settings: \n{}'.format(settings_dump))
@@ -154,16 +166,22 @@ async def app_setup(app, settings):
async def app_setup_basic(app, settings):
stack = AsyncExitStack()
- app['start_event_trackers'] = {
- 'static': trackers.general.static_start_event_tracker,
- 'cropped': partial(trackers.general.wrapped_tracker_start_event, trackers.general.cropped_tracker_start),
- 'filter_inaccurate': partial(trackers.general.wrapped_tracker_start_event, trackers.general.filter_inaccurate_tracker_start),
+ app["start_event_trackers"] = {
+ "static": trackers.general.static_start_event_tracker,
+ "cropped": partial(
+ trackers.general.wrapped_tracker_start_event,
+ trackers.general.cropped_tracker_start,
+ ),
+ "filter_inaccurate": partial(
+ trackers.general.wrapped_tracker_start_event,
+ trackers.general.filter_inaccurate_tracker_start,
+ ),
}
- app['trackers.settings'] = settings
- app['trackers.data_repo'] = stack.enter_context(dulwich.repo.Repo(settings['data_path']))
- app['trackers.events'] = {}
- app['analyse_processing_lock'] = asyncio.Lock()
+ app["trackers.settings"] = settings
+ app["trackers.data_repo"] = stack.enter_context(dulwich.repo.Repo(settings["data_path"]))
+ app["trackers.events"] = {}
+ app["analyse_processing_lock"] = asyncio.Lock()
return stack
@@ -178,15 +196,15 @@ async def run_with_app(settings, args):
run_fut = asyncio.ensure_future(func(app, settings, args))
def cancel(signal):
- logging.info('Canceling')
+ logging.info("Canceling")
run_fut.cancel()
- for signame in ('SIGINT', 'SIGTERM'):
+ for signame in ("SIGINT", "SIGTERM"):
loop.add_signal_handler(getattr(signal, signame), cancel, None)
try:
await run_fut
finally:
- for signame in ('SIGINT', 'SIGTERM'):
+ for signame in ("SIGINT", "SIGTERM"):
loop.remove_signal_handler(getattr(signal, signame))
@wraps(func)
@@ -199,30 +217,31 @@ def async_command_wrapper():
try:
loop.run_until_complete(run_with_app(settings, args))
except asyncio.CancelledError:
- logging.error('CancelledError.')
+ logging.error("CancelledError.")
return async_command_wrapper
+
return async_command_decorator
def event_command_parser(*args, **kwargs):
parser = get_base_argparser(*args, **kwargs)
- parser.add_argument('event_name', action='store')
+ parser.add_argument("event_name", action="store")
return parser
def event_name_clean(event_name, settings):
- if os.path.split(event_name)[0] != '':
- events_path = join(settings['data_path'], 'events')
+ if os.path.split(event_name)[0] != "":
+ events_path = join(settings["data_path"], "events")
events_rel_path = relpath(event_name, start=events_path)
- if os.path.split(events_rel_path)[0] == '':
+ if os.path.split(events_rel_path)[0] == "":
return events_rel_path
return event_name
@async_command(partial(event_command_parser, description="Convert live trackers to static data."))
async def convert_to_static(app, settings, args):
- tree_writer = TreeWriter(app['trackers.data_repo'])
+ tree_writer = TreeWriter(app["trackers.data_repo"])
event_name = event_name_clean(args.event_name, settings)
event = await trackers.events.Event.load(app, event_name, tree_writer)
@@ -230,48 +249,63 @@ async def convert_to_static(app, settings, args):
await event.store_analyse(tree_writer)
-@async_command(partial(event_command_parser, description="Recalculate and store analyse trackers"), basic=True)
+@async_command(
+ partial(event_command_parser, description="Recalculate and store analyse trackers"),
+ basic=True,
+)
async def store_analyse(app, settings, args):
- tree_writer = TreeWriter(app['trackers.data_repo'])
+ tree_writer = TreeWriter(app["trackers.data_repo"])
event_name = event_name_clean(args.event_name, settings)
event = await trackers.events.Event.load(app, event_name, tree_writer)
await event.store_analyse(tree_writer)
-@async_command(partial(event_command_parser, description="Assigns unique colors to riders"), basic=True)
+@async_command(
+ partial(event_command_parser, description="Assigns unique colors to riders"),
+ basic=True,
+)
async def assign_rider_colors(app, settings, args):
event_name = event_name_clean(args.event_name, settings)
- tree_writer = TreeWriter(app['trackers.data_repo'])
+ tree_writer = TreeWriter(app["trackers.data_repo"])
event = await trackers.events.Event.load(app, event_name, tree_writer)
assign_rider_colors_inner(event)
- await event.save(f'{event_name}: assign_rider_colors', tree_writer=tree_writer)
+ await event.save(f"{event_name}: assign_rider_colors", tree_writer=tree_writer)
def assign_rider_colors_inner(event):
- num_riders = len(event.config['riders'])
+ num_riders = len(event.config["riders"])
hues = [round(i * 360 / num_riders) for i in range(num_riders)]
alternating_chunks = floor(sqrt(num_riders))
chunked_hues = [list(s) for s in chunked(hues, alternating_chunks)]
alternating_hues = interleave_longest(*chunked_hues)
- for rider, hue in zip(event.config['riders'], alternating_hues):
- rider['color'] = 'hsl({}, 100%, 50%)'.format(hue)
- rider['color_pre_post'] = 'hsl({}, 100%, 70%)'.format(hue)
- rider['color_marker'] = 'hsl({}, 100%, 60%)'.format(hue)
+ for rider, hue in zip(event.config["riders"], alternating_hues):
+ rider["color"] = "hsl({}, 100%, 50%)".format(hue)
+ rider["color_pre_post"] = "hsl({}, 100%, 70%)".format(hue)
+ rider["color_marker"] = "hsl({}, 100%, 60%)".format(hue)
def add_gpx_to_event_routes_parser():
parser = get_base_argparser(description="Add a gpx file to the routes for of an event.")
- parser.add_argument('event_name', action='store')
- parser.add_argument('gpx_file', action='store')
- parser.add_argument('--no-elevation', action='store_true')
- parser.add_argument('--split-at-dist', action='store', type=int, nargs='*',
- help="Distances to split route at when performing RDP simplification")
- parser.add_argument('--split-point-range', action='store', type=int, default=500)
- parser.add_argument('--rdp-epsilon', action='store', type=int, default=2)
- parser.add_argument('--circular-range', action='store', type=int,
- help="Set to about 1/2 of distance (m) of circular route. To help with find closest point.")
- parser.add_argument('--print', action='store_true')
- parser.add_argument('--replace-main', action='store_true')
+ parser.add_argument("event_name", action="store")
+ parser.add_argument("gpx_file", action="store")
+ parser.add_argument("--no-elevation", action="store_true")
+ parser.add_argument(
+ "--split-at-dist",
+ action="store",
+ type=int,
+ nargs="*",
+ help="Distances to split route at when performing RDP simplification",
+ )
+ parser.add_argument("--split-point-range", action="store", type=int, default=500)
+ parser.add_argument("--rdp-epsilon", action="store", type=int, default=2)
+ parser.add_argument(
+ "--circular-range",
+ action="store",
+ type=int,
+ help="Set to about 1/2 of distance (m) of circular route. To help with find closest point.",
+ )
+ parser.add_argument("--print", action="store_true")
+ parser.add_argument("--replace-main", action="store_true")
return parser
@@ -286,35 +320,46 @@ async def add_gpx_to_event_routes(app, settings, args):
xml_doc = xml.fromstring(gpx_text)
gpx_ns = {
- '1.0': {'gpx': 'http://www.topografix.com/GPX/1/0', },
- '1.1': {'gpx': 'http://www.topografix.com/GPX/1/1', },
- }[xml_doc.attrib['version']]
-
- trkpts = xml_doc.findall('./gpx:trk/gpx:trkseg/gpx:trkpt', gpx_ns)
- points = [[float(trkpt.attrib['lat']), float(trkpt.attrib['lon'])] for trkpt in trkpts]
-
- route = {'original_points': points}
- for key in ('no_elevation', 'split_at_dist', 'split_point_range', 'rdp_epsilon', 'circular_range', 'gpx_file'):
+ "1.0": {
+ "gpx": "http://www.topografix.com/GPX/1/0",
+ },
+ "1.1": {
+ "gpx": "http://www.topografix.com/GPX/1/1",
+ },
+ }[xml_doc.attrib["version"]]
+
+ trkpts = xml_doc.findall("./gpx:trk/gpx:trkseg/gpx:trkpt", gpx_ns)
+ points = [[float(trkpt.attrib["lat"]), float(trkpt.attrib["lon"])] for trkpt in trkpts]
+
+ route = {"original_points": points}
+ for key in (
+ "no_elevation",
+ "split_at_dist",
+ "split_point_range",
+ "rdp_epsilon",
+ "circular_range",
+ "gpx_file",
+ ):
route[key] = getattr(args, key)
markers = [
{
- 'title': wpt.find('gpx:name', gpx_ns).text,
- 'svg_marker': {
- 'direction': 'sw',
- 'text': wpt.find('gpx:name', gpx_ns).text,
+ "title": wpt.find("gpx:name", gpx_ns).text,
+ "svg_marker": {
+ "direction": "sw",
+ "text": wpt.find("gpx:name", gpx_ns).text,
},
- 'position': {
- 'lat': round(float(wpt.attrib['lat']), 6),
- 'lng': round(float(wpt.attrib['lon']), 6),
+ "position": {
+ "lat": round(float(wpt.attrib["lat"]), 6),
+ "lng": round(float(wpt.attrib["lon"]), 6),
},
}
- for wpt in xml_doc.findall('./gpx:wpt', gpx_ns)
+ for wpt in xml_doc.findall("./gpx:wpt", gpx_ns)
]
event_name = event_name_clean(args.event_name, settings)
if not args.print:
- writer = TreeWriter(app['trackers.data_repo'])
+ writer = TreeWriter(app["trackers.data_repo"])
event = await trackers.events.Event.load(app, event_name, writer)
await process_route(settings, route)
if args.replace_main:
@@ -322,124 +367,172 @@ async def add_gpx_to_event_routes(app, settings, args):
else:
event.routes.append(route)
process_secondary_route_details(event.routes)
- if 'markers' not in event.config:
- event.config['markers'] = []
- event.config['markers'].extend(markers)
+ if "markers" not in event.config:
+ event.config["markers"] = []
+ event.config["markers"].extend(markers)
# TODO - add gpx file to repo
- await event.save(f'{event_name}: add_gpx_to_event_routes {args.gpx_file}',
- tree_writer=writer, save_routes=True)
+ await event.save(
+ f"{event_name}: add_gpx_to_event_routes {args.gpx_file}",
+ tree_writer=writer,
+ save_routes=True,
+ )
else:
- original_points = route['original_points']
- filtered_points = (point for last_point, point in zip([None] + original_points[:-1], original_points) if point != last_point)
+ original_points = route["original_points"]
+ filtered_points = (
+ point
+ for last_point, point in zip([None] + original_points[:-1], original_points)
+ if point != last_point
+ )
point_points = route_with_distance_and_index(filtered_points)
for point in point_points:
- print(f'{point.index}: {point.distance} {point.lat},{point.lng}')
+ print(f"{point.index}: {point.distance} {point.lat},{point.lng}")
-@async_command(partial(event_command_parser, description="Open and save event. Side effect is convert to new formats."), basic=True)
+@async_command(
+ partial(
+ event_command_parser,
+ description="Open and save event. Side effect is convert to new formats.",
+ ),
+ basic=True,
+)
async def reformat_event(app, settings, args):
event_name = event_name_clean(args.event_name, settings)
- writer = TreeWriter(app['trackers.data_repo'])
+ writer = TreeWriter(app["trackers.data_repo"])
event = await trackers.events.Event.load(app, event_name, writer)
for route in event.routes:
- for key in ('start_distance', 'end_distance', 'dist_factor'):
+ for key in ("start_distance", "end_distance", "dist_factor"):
if key in route:
route[key] = float(route[key])
- await event.save(f'{event_name}: reformat_event', tree_writer=writer, save_routes=True)
+ await event.save(f"{event_name}: reformat_event", tree_writer=writer, save_routes=True)
@async_command(partial(event_command_parser, description="Update bounds for event"), basic=True)
async def update_bounds(app, settings, args):
event_name = event_name_clean(args.event_name, settings)
- writer = TreeWriter(app['trackers.data_repo'])
+ writer = TreeWriter(app["trackers.data_repo"])
event = await trackers.events.Event.load(app, event_name, writer)
update_bounds_inner(event)
- await event.save(f'{event_name}: update_bounds', tree_writer=writer)
+ await event.save(f"{event_name}: update_bounds", tree_writer=writer)
def update_bounds_inner(event):
- points = list(itertools.chain.from_iterable(
- [((marker['position']['lat'], marker['position']['lng']), ) for marker in event.config.get('markers', ())] +
- [route['points'] for route in event.routes]
- ))
+ points = list(
+ itertools.chain.from_iterable(
+ [
+ ((marker["position"]["lat"], marker["position"]["lng"]),)
+ for marker in event.config.get("markers", ())
+ ]
+ + [route["points"] for route in event.routes]
+ )
+ )
lats = [point[0] for point in points]
lngs = [point[1] for point in points]
- event.config['bounds'] = {
- 'north': max(lats),
- 'south': min(lats),
- 'east': max(lngs),
- 'west': min(lngs),
+ event.config["bounds"] = {
+ "north": max(lats),
+ "south": min(lats),
+ "east": max(lngs),
+ "west": min(lngs),
}
def process_event_routes_parser(*args, **kwargs):
parser = get_base_argparser(*args, **kwargs)
- parser.add_argument('event_name', action='store')
- parser.add_argument('--rdp-epsilon', action='store', type=int)
+ parser.add_argument("event_name", action="store")
+ parser.add_argument("--rdp-epsilon", action="store", type=int)
return parser
-@async_command(partial(process_event_routes_parser, description="Reprocess event routes."), basic=True)
+@async_command(
+ partial(process_event_routes_parser, description="Reprocess event routes."),
+ basic=True,
+)
async def process_event_routes(app, settings, args):
event_name = event_name_clean(args.event_name, settings)
- writer = TreeWriter(app['trackers.data_repo'])
+ writer = TreeWriter(app["trackers.data_repo"])
event = await trackers.events.Event.load(app, event_name, writer)
for route in event.routes:
if args.rdp_epsilon:
- route['rdp_epsilon'] = args.rdp_epsilon
+ route["rdp_epsilon"] = args.rdp_epsilon
await process_route(settings, route)
process_secondary_route_details(event.routes)
- await event.save(f'{event_name}: process_event_routes', tree_writer=writer, save_routes=True)
+ await event.save(f"{event_name}: process_event_routes", tree_writer=writer, save_routes=True)
def process_secondary_route_details(routes):
main_route = get_analyse_route(routes[0])
for i, route in enumerate(routes):
if i > 0:
- route['main'] = False
- route_points = route_with_distance_and_index(route['points'])
+ route["main"] = False
+ route_points = route_with_distance_and_index(route["points"])
start_closest = find_closest_point_pair_route(main_route, route_points[0], None, None)
prev_point = start_closest.point_pair[0]
- route['prev_point'] = prev_point.index
- route['start_distance'] = start_distance = float(prev_point.distance + distance(prev_point, route_points[0]))
+ route["prev_point"] = prev_point.index
+ route["start_distance"] = start_distance = float(
+ prev_point.distance + distance(prev_point, route_points[0])
+ )
end_closest = find_closest_point_pair_route(main_route, route_points[-1], None, None)
next_point = end_closest.point_pair[1]
- route['next_point'] = next_point.index
- route['end_distance'] = end_distance = float(next_point.distance - distance(next_point, route_points[-1]))
- route['dist_factor'] = float((end_distance - start_distance) / route_points[-1].distance)
+ route["next_point"] = next_point.index
+ route["end_distance"] = end_distance = float(
+ next_point.distance - distance(next_point, route_points[-1])
+ )
+ route["dist_factor"] = float(
+ (end_distance - start_distance) / route_points[-1].distance
+ )
else:
- route['main'] = True
+ route["main"] = True
async def process_route(settings, route):
- original_points = route['original_points']
- filtered_points = (point for last_point, point in zip([None] + original_points[:-1], original_points) if point != last_point)
+ original_points = route["original_points"]
+ filtered_points = (
+ point
+ for last_point, point in zip([None] + original_points[:-1], original_points)
+ if point != last_point
+ )
point_points = route_with_distance_and_index(filtered_points)
- if 'rdp_epsilon' not in route:
- route['rdp_epsilon'] = 2
+ if "rdp_epsilon" not in route:
+ route["rdp_epsilon"] = 2
- if not route.get('split_at_dist'):
- points = ramer_douglas_peucker(point_points, route['rdp_epsilon'])
+ if not route.get("split_at_dist"):
+ points = ramer_douglas_peucker(point_points, route["rdp_epsilon"])
else:
- points = ramer_douglas_peucker_sections(point_points, route['rdp_epsilon'], route['split_at_dist'], route['split_point_range'])
- route['points'] = [(point.lat, point.lng) for point in points]
-
- indexed_points = [IndexedPoint(point.lat, point.lng, index=i, distance=point.distance) for i, point in enumerate(points)]
- if not route.get('split_at_dist'):
+ points = ramer_douglas_peucker_sections(
+ point_points,
+ route["rdp_epsilon"],
+ route["split_at_dist"],
+ route["split_point_range"],
+ )
+ route["points"] = [(point.lat, point.lng) for point in points]
+
+ indexed_points = [
+ IndexedPoint(point.lat, point.lng, index=i, distance=point.distance)
+ for i, point in enumerate(points)
+ ]
+ if not route.get("split_at_dist"):
simplified_points = ramer_douglas_peucker(indexed_points, 500)
else:
- simplified_points = ramer_douglas_peucker_sections(indexed_points, 500, route['split_at_dist'], route['split_point_range'])
- route['simplified_points_indexes'] = [point.index for point in simplified_points]
+ simplified_points = ramer_douglas_peucker_sections(
+ indexed_points, 500, route["split_at_dist"], route["split_point_range"]
+ )
+ route["simplified_points_indexes"] = [point.index for point in simplified_points]
- logging.info(f"Original point count: {len(route['original_points'])}, point count: {len(points)}, simplified point count: {len(simplified_points)}")
+ logging.info(
+ f"Original point count: {len(route['original_points'])}, point count: {len(points)}, simplified point count: {len(simplified_points)}"
+ )
- if not route.get('no_elevation', False):
- logging.info('Getting Elevation')
+ if not route.get("no_elevation", False):
+ logging.info("Getting Elevation")
elevation_points = list(get_equal_spaced_points(points, 500))
- elevations = await get_elevation_for_points(settings, [point for point, distance in elevation_points])
- route['elevation'] = [(round(point.lat, 6), round(point.lng, 6), elevation, dist) for elevation, (point, dist) in zip(elevations, elevation_points)]
+ elevations = await get_elevation_for_points(
+ settings, [point for point, distance in elevation_points]
+ )
+ route["elevation"] = [
+ (round(point.lat, 6), round(point.lng, 6), elevation, dist)
+ for elevation, (point, dist) in zip(elevations, elevation_points)
+ ]
async def get_elevation_for_points(settings, points):
@@ -447,70 +540,85 @@ async def get_elevation_for_points(settings, points):
result = []
async with aiohttp.ClientSession() as session:
for i in range(0, len(points), n):
- section_points = points[i:i + n]
+ section_points = points[i : i + n]
- section_polyline = polyline.encode([(round(point.lat, 6), round(point.lng, 6)) for point in section_points])
+ section_polyline = polyline.encode(
+ [(round(point.lat, 6), round(point.lng, 6)) for point in section_points]
+ )
r = await session.get(
- 'https://maps.googleapis.com/maps/api/elevation/json',
+ "https://maps.googleapis.com/maps/api/elevation/json",
params={
- 'sensor': 'false',
- 'key': settings['google_api_key'],
- 'locations': "enc:{}".format(section_polyline)
- })
+ "sensor": "false",
+ "key": settings["google_api_key"],
+ "locations": "enc:{}".format(section_polyline),
+ },
+ )
elevations = await r.json()
- if elevations['status'] != 'OK':
+ if elevations["status"] != "OK":
logging.error(elevations)
else:
- result.extend((elv['elevation'] for elv in elevations['results']))
+ result.extend((elv["elevation"] for elv in elevations["results"]))
return result
-@async_command(partial(event_command_parser, description="Loads csv from stdin, writes to riders in config"), basic=True)
+@async_command(
+ partial(
+ event_command_parser,
+ description="Loads csv from stdin, writes to riders in config",
+ ),
+ basic=True,
+)
async def load_riders_from_csv(app, settings, args):
event_name = event_name_clean(args.event_name, settings)
- tree_writer = TreeWriter(app['trackers.data_repo'])
+ tree_writer = TreeWriter(app["trackers.data_repo"])
event = await trackers.events.Event.load(app, event_name, tree_writer)
import csv
+
reader = csv.DictReader(sys.stdin)
def trackers_from_row(row):
- if row.get('Traccar Device Id'):
- yield {'type': 'traccar', 'unique_id': row['Traccar Device Id']}
- if row.get('TKStorage'):
- yield {'type': 'tkstorage', 'id': row['TKStorage']}
- if row.get('Spot'):
- yield {'type': 'spot', 'feed_id': row['Spot']}
-
- event.config['riders'] = [
+ if row.get("Traccar Device Id"):
+ yield {"type": "traccar", "unique_id": row["Traccar Device Id"]}
+ if row.get("TKStorage"):
+ yield {"type": "tkstorage", "id": row["TKStorage"]}
+ if row.get("Spot"):
+ yield {"type": "spot", "feed_id": row["Spot"]}
+
+ event.config["riders"] = [
{
- 'name': row['Name'],
- 'name_short': row['Short Name'],
- 'trackers': list(trackers_from_row(row)),
+ "name": row["Name"],
+ "name_short": row["Short Name"],
+ "trackers": list(trackers_from_row(row)),
}
for row in reader
]
assign_rider_colors_inner(event)
- await event.save(f'{event_name}: load_riders_from_csv', tree_writer=tree_writer)
+ await event.save(f"{event_name}: load_riders_from_csv", tree_writer=tree_writer)
-@async_command(partial(event_command_parser, description="Runs analyse trackers. Just for testing."))
+@async_command(
+ partial(event_command_parser, description="Runs analyse trackers. Just for testing.")
+)
async def analyse(app, settings, args):
event_name = event_name_clean(args.event_name, settings)
- tree_writer = TreeWriter(app['trackers.data_repo'])
+ tree_writer = TreeWriter(app["trackers.data_repo"])
event = await trackers.events.Event.load(app, event_name, tree_writer)
await event.start_trackers(analyse=True)
await event.stop_and_complete_trackers()
-@async_command(partial(event_command_parser, description="Print riders names and trackers"), basic=True)
+@async_command(
+ partial(event_command_parser, description="Print riders names and trackers"),
+ basic=True,
+)
async def print_names_and_trackers(app, settings, args):
event_name = event_name_clean(args.event_name, settings)
- tree_reader = TreeReader(app['trackers.data_repo'])
+ tree_reader = TreeReader(app["trackers.data_repo"])
event = await trackers.events.Event.load(app, event_name, tree_reader)
- for rider in event.config['riders']:
- tracker_ids = [tracker.get('id', '') for tracker in rider.get('trackers', ())]
+ for rider in event.config["riders"]:
+ tracker_ids = [tracker.get("id", "") for tracker in rider.get("trackers", ())]
for tracker in tracker_ids:
print(f'{tracker}\t{rider["name"]}')
diff --git a/trackers/client_test_tools.py b/trackers/client_test_tools.py
index fc3e584..36117cc 100644
--- a/trackers/client_test_tools.py
+++ b/trackers/client_test_tools.py
@@ -8,34 +8,28 @@
import subprocess
import sys
import tempfile
-from contextlib import asynccontextmanager, AsyncExitStack, closing, suppress
+from contextlib import AsyncExitStack, asynccontextmanager, closing, suppress
from functools import partial
-import aionotify
import arsenic
import pkg_resources
import yaml
from aiohttp import web
+from asyncinotify import Inotify, Mask
from trackers.tests import web_server_fixture
from trackers.web_app import convert_client_urls_to_paths
-
log = logging.getLogger(__name__)
@asynccontextmanager
-async def watch_path(loop, path):
- watcher = aionotify.Watcher()
- watcher.watch(path=path, flags=aionotify.Flags.MODIFY)
-
- # TODO: ideally need a recursive setup.
- watcher.watch(path=os.path.join(path, 'tests'), flags=aionotify.Flags.MODIFY)
- await watcher.setup(loop)
- try:
- yield watcher
- finally:
- watcher.close()
+async def watch_path(path):
+ with Inotify() as inotify:
+ inotify.add_watch(path, Mask.MODIFY)
+ # TODO: ideally need a recursive setup.
+ inotify.add_watch(os.path.join(path, "tests"), Mask.MODIFY)
+ yield inotify
@asynccontextmanager
@@ -43,7 +37,7 @@ async def on_signals_set_event(loop, signals):
event = asyncio.Event()
def handler(signame):
- log.info(f'{signame} received')
+ log.info(f"{signame} received")
event.set()
for signame in signals:
@@ -56,8 +50,8 @@ def handler(signame):
def qunit_runner():
- parser = argparse.ArgumentParser(description='Run qunit tests. Watch for changes.')
- parser.add_argument('-c', '--coverage', action='store_true')
+ parser = argparse.ArgumentParser(description="Run qunit tests. Watch for changes.")
+ parser.add_argument("-c", "--coverage", action="store_true")
args = parser.parse_args()
logging.basicConfig(stream=sys.stdout)
with closing(asyncio.get_event_loop()) as loop:
@@ -66,17 +60,18 @@ def qunit_runner():
async def qunit_runner_async(args, loop):
async with AsyncExitStack() as stack:
-
- org_static_path = pkg_resources.resource_filename('trackers', '/static')
+ org_static_path = pkg_resources.resource_filename("trackers", "/static")
if args.coverage:
- static_path = os.path.join(await stack.enter_async_context(tempfile.TemporaryDirectory()), 'static')
+ static_path = os.path.join(
+ await stack.enter_async_context(tempfile.TemporaryDirectory()), "static"
+ )
os.mkdir(static_path)
else:
static_path = org_static_path
app = web.Application()
- app.router.add_static('/static', static_path)
+ app.router.add_static("/static", static_path)
async def receive_log(request):
text = await request.text()
@@ -86,24 +81,31 @@ async def receive_log(request):
outfile = sys.stderr
outfile.write(text)
else:
- if isinstance(result, dict) and 'source' in result:
- result['source'] = literal_str(convert_client_urls_to_paths(static_path, result['source']).strip().strip('@').strip('\n'))
- outfile = sys.stderr if isinstance(result, dict) and result.get('failed') else sys.stdout
+ if isinstance(result, dict) and "source" in result:
+ result["source"] = literal_str(
+ convert_client_urls_to_paths(static_path, result["source"])
+ .strip()
+ .strip("@")
+ .strip("\n")
+ )
+ outfile = (
+ sys.stderr if isinstance(result, dict) and result.get("failed") else sys.stdout
+ )
yaml.dump(result, outfile, default_flow_style=False, Dumper=DumperWithLiteral)
- outfile.write('---\n')
- return web.Response(text='Thanks browser.')
+ outfile.write("---\n")
+ return web.Response(text="Thanks browser.")
- app.router.add_route('POST', '/results', handler=receive_log, name='receive_result')
- app.router.add_route('POST', '/log', handler=receive_log, name='receive_log')
+ app.router.add_route("POST", "/results", handler=receive_log, name="receive_result")
+ app.router.add_route("POST", "/log", handler=receive_log, name="receive_log")
- app.router.add_route('POST', '/coverage', handler=receive_coverage, name='receive_coverage')
+ app.router.add_route("POST", "/coverage", handler=receive_coverage, name="receive_coverage")
async def receive_error(request):
body = await request.text()
- sys.stderr.write(body + '\n')
- return web.Response(text='Thanks browser.')
+ sys.stderr.write(body + "\n")
+ return web.Response(text="Thanks browser.")
- app.router.add_route('POST', '/error', handler=receive_error, name='receive_error')
+ app.router.add_route("POST", "/error", handler=receive_error, name="receive_error")
url = await stack.enter_async_context(web_server_fixture(loop, app))
@@ -112,27 +114,34 @@ async def receive_error(request):
driver = await stack.enter_async_context(arsenic.get_session(service, browser))
if args.coverage:
- app['coverage_driver'] = await stack.enter_async_context(arsenic.get_session(service, browser))
+ app["coverage_driver"] = await stack.enter_async_context(
+ arsenic.get_session(service, browser)
+ )
+
+ inotify = await stack.enter_async_context(watch_path(org_static_path))
+ inotify_iter = aiter(inotify)
- watcher = await stack.enter_async_context(watch_path(loop, org_static_path))
- stop_event = await stack.enter_async_context(on_signals_set_event(loop, ('SIGINT', 'SIGTERM')))
+ stop_event = await stack.enter_async_context(
+ on_signals_set_event(loop, ("SIGINT", "SIGTERM"))
+ )
stop_event_wait = asyncio.ensure_future(stop_event.wait())
while not stop_event.is_set():
if args.coverage:
- make_instrumented_static(org_static_path, static_path, (
- 'lib.js',
- ))
+ make_instrumented_static(org_static_path, static_path, ("lib.js",))
- await driver.get(f'{url}/static/tests/test-lib.html#post_results')
+ await driver.get(f"{url}/static/tests/test-lib.html#post_results")
- get_watcher_event = asyncio.ensure_future(watcher.get_event())
- await asyncio.wait((get_watcher_event, stop_event_wait), return_when=asyncio.FIRST_COMPLETED)
+ get_watcher_event = asyncio.ensure_future(aiter(inotify_iter))
+ await asyncio.wait(
+ (get_watcher_event, stop_event_wait),
+ return_when=asyncio.FIRST_COMPLETED,
+ )
if not stop_event.is_set():
await get_watcher_event
await asyncio.sleep(1)
- await driver.get('about:blank')
+ await driver.get("about:blank")
else:
get_watcher_event.cancel()
with suppress(asyncio.CancelledError):
@@ -144,7 +153,7 @@ class literal_str(str):
def literal_str_representer(dumper, data):
- return dumper.represent_scalar(u'tag:yaml.org,2002:str', data, style='|')
+ return dumper.represent_scalar("tag:yaml.org,2002:str", data, style="|")
class DumperWithLiteral(yaml.Dumper):
@@ -159,25 +168,35 @@ def make_instrumented_static(src_path, dest_path, instrumented_files):
shutil.copytree(src_path, dest_path)
for item in instrumented_files:
- subprocess.check_call([
- os.path.abspath('node_modules/nyc/bin/nyc.js'), 'instrument',
- item, dest_path,
- '--produce-source-map', 'true',
- ], cwd=src_path)
+ subprocess.check_call(
+ [
+ os.path.abspath("node_modules/nyc/bin/nyc.js"),
+ "instrument",
+ item,
+ dest_path,
+ "--produce-source-map",
+ "true",
+ ],
+ cwd=src_path,
+ )
async def receive_coverage(request):
coverage = await request.text()
- src_path = pkg_resources.resource_filename('trackers', '/static')
- nyc = os.path.abspath('node_modules/nyc/bin/nyc.js')
+ src_path = pkg_resources.resource_filename("trackers", "/static")
+ nyc = os.path.abspath("node_modules/nyc/bin/nyc.js")
with tempfile.TemporaryDirectory() as tempdir:
-
- with open(os.path.join(tempdir, 'out.json'), 'w') as f:
+ with open(os.path.join(tempdir, "out.json"), "w") as f:
f.write(coverage)
- subprocess.check_call([nyc, 'report', '--temp-directory', tempdir, src_path], cwd=src_path)
- subprocess.check_call([nyc, 'report', '--temp-directory', tempdir, '--reporter', 'html'], cwd=src_path)
+ subprocess.check_call([nyc, "report", "--temp-directory", tempdir, src_path], cwd=src_path)
+ subprocess.check_call(
+ [nyc, "report", "--temp-directory", tempdir, "--reporter", "html"],
+ cwd=src_path,
+ )
- await request.app['coverage_driver'].get('file://{}'.format(os.path.join(src_path, 'coverage/index.html')))
+ await request.app["coverage_driver"].get(
+ "file://{}".format(os.path.join(src_path, "coverage/index.html"))
+ )
- return web.Response(text='Thanks browser.')
+ return web.Response(text="Thanks browser.")
diff --git a/trackers/combined.py b/trackers/combined.py
index 9744c35..a08db60 100644
--- a/trackers/combined.py
+++ b/trackers/combined.py
@@ -4,14 +4,17 @@
def time_key(item):
- return item.get('time') or item.get('server_time')
+ return item.get("time") or item.get("server_time")
class Combined(Tracker):
-
@classmethod
async def start(cls, name, trackers, new_points_callbacks=(), reset_points_callbacks=()):
- tracker = cls(name, new_points_callbacks=new_points_callbacks, reset_points_callbacks=reset_points_callbacks)
+ tracker = cls(
+ name,
+ new_points_callbacks=new_points_callbacks,
+ reset_points_callbacks=reset_points_callbacks,
+ )
tracker.trackers = list(trackers)
tracker.sub_to_be_completed = len(tracker.trackers)
@@ -38,7 +41,12 @@ async def append_sub_tracker(self, tracker):
tracker.completed.add_done_callback(self.on_sub_completed)
def get_sorted_points(self):
- return list(sorted(chain.from_iterable((sub_tracker.points for sub_tracker in self.trackers)), key=time_key))
+ return list(
+ sorted(
+ chain.from_iterable((sub_tracker.points for sub_tracker in self.trackers)),
+ key=time_key,
+ )
+ )
async def on_sub_new_points(self, sub_tracker, points):
if points:
@@ -47,7 +55,7 @@ async def on_sub_new_points(self, sub_tracker, points):
if is_sorted:
await self.new_points(points)
else:
- self.logger.debug('Points not sorted. Resting.')
+ self.logger.debug("Points not sorted. Resting.")
all_points_sorted = list(sorted(chain(self.points, points), key=time_key))
await self.reset_points()
await self.new_points(all_points_sorted)
diff --git a/trackers/contrib/dataclass_tools.py b/trackers/contrib/dataclass_tools.py
index 87f73a4..c171a08 100644
--- a/trackers/contrib/dataclass_tools.py
+++ b/trackers/contrib/dataclass_tools.py
@@ -8,21 +8,21 @@ def add_slots(cls):
# after a class has been created.
# Make sure __slots__ isn't already set.
- if '__slots__' in cls.__dict__:
- raise TypeError(f'{cls.__name__} already specifies __slots__')
+ if "__slots__" in cls.__dict__:
+ raise TypeError(f"{cls.__name__} already specifies __slots__")
# Create a new dict for our new class.
cls_dict = dict(cls.__dict__)
field_names = tuple(f.name for f in dataclasses.fields(cls))
- cls_dict['__slots__'] = field_names
+ cls_dict["__slots__"] = field_names
for field_name in field_names:
# Remove our attributes, if present. They'll still be
# available in _MARKER.
cls_dict.pop(field_name, None)
# Remove __dict__ itself.
- cls_dict.pop('__dict__', None)
+ cls_dict.pop("__dict__", None)
# And finally create the class.
- qualname = getattr(cls, '__qualname__', None)
+ qualname = getattr(cls, "__qualname__", None)
cls = type(cls)(cls.__name__, cls.__bases__, cls_dict)
if qualname is not None:
cls.__qualname__ = qualname
diff --git a/trackers/dulwich_helpers.py b/trackers/dulwich_helpers.py
deleted file mode 100644
index 75902e7..0000000
--- a/trackers/dulwich_helpers.py
+++ /dev/null
@@ -1,153 +0,0 @@
-import logging
-import stat
-import subprocess
-from contextlib import suppress
-from time import time, timezone
-
-from dulwich.errors import NotTreeError
-from dulwich.objects import Blob, Commit, Tree
-from dulwich.objectspec import parse_tree
-
-
-class TreeReader(object):
-
- def __init__(self, repo, treeish='HEAD', encoding="UTF-8"):
- self.repo = repo
- self.treeish = treeish
- self.tree = None
- self.lookup_obj = repo.__getitem__
- self.encoding = encoding
- self.reset()
-
- def reset(self):
- self.tree = parse_tree(self.repo, self.treeish)
-
- def lookup(self, path):
- return self.tree.lookup_path(self.lookup_obj, path.encode(self.encoding))
-
- def get(self, path):
- _, sha = self.tree.lookup_path(self.lookup_obj, path.encode(self.encoding))
- return self.lookup_obj(sha)
-
- def tree_items(self, path):
- tree = self.get(path)
- if not isinstance(tree, Tree):
- raise NotTreeError(path)
- return [item.decode(self.encoding) for item in tree]
-
- def exists(self, path):
- try:
- self.lookup(path)
- except KeyError:
- return False
- else:
- return True
-
-
-class TreeWriter(TreeReader):
-
- # TODO: changed_objects should have a ref count
-
- def __init__(self, repo, branch=b'HEAD', encoding="UTF-8"):
- self.repo = repo
- self.encoding = encoding
- self.branch = branch
- self.reset()
-
- def reset(self):
- try:
- self.org_commit_id = self.repo.refs[self.branch]
- except KeyError:
- self.org_commit_id = None
- self.tree = Tree()
- else:
- self.tree = parse_tree(self.repo, self.org_commit_id)
- self.org_tree_id = self.tree.id
- self.changed_objects = {}
-
- def lookup_obj(self, sha):
- try:
- return self.changed_objects[sha]
- except KeyError:
- return self.repo[sha]
-
- def set(self, path, obj, mode):
- path_items = path.encode(self.encoding).split(b'/')
- sub_tree = self.tree
- old_trees = [sub_tree]
- for name in path_items[:-1]:
- try:
- _, sub_tree_sha = sub_tree[name]
- except KeyError:
- sub_tree = Tree()
- else:
- sub_tree = self.lookup_obj(sub_tree_sha)
- old_trees.append(sub_tree)
-
- new_objs = []
- for old_tree, name in reversed(tuple(zip(old_trees, path_items))):
- new_tree = old_tree.copy()
- if obj is None or obj.id == b'4b825dc642cb6eb9a060e54bf8d69288fbee4904':
- if name not in new_tree:
- raise KeyError(name)
- del new_tree[name]
- # print(f'del old: {old_tree} new: {new_tree} name: {name}')
- else:
- obj_id = obj.id
- new_objs.append(obj)
- new_tree[name] = (mode, obj_id)
- # print(f'set old: {old_tree} new: {new_tree} name: {name} obj_id: {obj_id}')
-
- obj = new_tree
- mode = stat.S_IFDIR
-
- new_objs.append(obj)
- self.tree = obj
-
- # print(f'old: {old_trees} new: {new_objs}')
- for old_tree in old_trees:
- if old_tree:
- with suppress(KeyError):
- del self.changed_objects[old_tree.id]
- for obj in new_objs:
- self.changed_objects[obj.id] = obj
-
- def set_data(self, path, data, mode=stat.S_IFREG | 0o644):
- obj = Blob()
- obj.data = data
- self.set(path, obj, mode)
- return obj
-
- def remove(self, path):
- self.set(path, None, None)
-
- def commit(self, message, author=None):
- commit = Commit()
- commit.tree = self.tree.id
- if author is None:
- config = self.repo.get_config_stack()
- author = self.repo._get_user_identity(config)
- else:
- author = author.encode(self.encoding)
- commit.author = commit.committer = author
- commit.commit_time = commit.author_time = int(time())
- tz = timezone
- commit.commit_timezone = commit.author_timezone = tz
- commit.message = message.encode(self.encoding)
- commit.encoding = self.encoding.encode('ascii')
- if self.org_commit_id:
- commit.parents = [self.org_commit_id]
-
- commit_id = commit.id
- self.changed_objects[commit_id] = commit
- self.repo.object_store.add_objects([(obj, None) for obj in self.changed_objects.values()])
- self.repo.refs.set_if_equals(self.branch, self.org_commit_id, commit_id)
-
- if hasattr(self.repo, 'has_index') and self.repo.has_index():
- # Apply patch to working tree.
- try:
- subprocess.call(['git', 'cherry-pick', commit_id, '--no-commit'], cwd=self.repo.path)
- except Exception:
- logging.getLogger(__name__).exception('Error updating working tree:')
-
- self.reset()
diff --git a/trackers/events.py b/trackers/events.py
index c68f408..79c9855 100644
--- a/trackers/events.py
+++ b/trackers/events.py
@@ -4,23 +4,35 @@
import os
from bisect import bisect
from collections import defaultdict
-from contextlib import closing, suppress
+from contextlib import suppress
from dataclasses import dataclass, field
from datetime import datetime, timedelta
from functools import partial
from itertools import chain
from typing import List
-import aionotify
import msgpack
import yaml
+from asyncinotify import Inotify, Mask
+from dulwich.repo import Repo
+from dulwich_tree import TreeReader, TreeWriter
from more_itertools import spy
from trackers.analyse import AnalyseTracker, get_analyse_routes
-from trackers.base import BlockedList, cancel_and_wait_task, general_fut_done_callback, Observable, Tracker
+from trackers.base import (
+ BlockedList,
+ Observable,
+ Tracker,
+ cancel_and_wait_task,
+ general_fut_done_callback,
+)
from trackers.combined import Combined
-from trackers.dulwich_helpers import TreeReader, TreeWriter
-from trackers.general import hash_bytes, index_and_hash_tracker, json_encode, start_replay_tracker
+from trackers.general import (
+ hash_bytes,
+ index_and_hash_tracker,
+ json_encode,
+ start_replay_tracker,
+)
from trackers.persisted_func_cache import PersistedFuncCache
logger = logging.getLogger(__name__)
@@ -30,67 +42,77 @@
# TODO: this is no longer specific to events. Move to somewhere
-async def load_with_watcher(app, ref=b'HEAD', **kwargs):
+async def load_with_watcher(app, ref=b"HEAD", **kwargs):
try:
await load(app, ref=ref, **kwargs)
- repo = app['trackers.data_repo']
-
- if hasattr(repo.refs, 'refpath'):
- while True:
- refnames, sha = repo.refs.follow(ref)
- paths = [repo.refs.refpath(ref) for ref in refnames]
- logger.debug(f'Watching paths {paths}')
-
- try:
- with closing(aionotify.Watcher()) as watcher:
- await watcher.setup(asyncio.get_event_loop())
- for path in paths:
- watcher.watch(path.decode(), flags=aionotify.Flags.MODIFY + aionotify.Flags.DELETE_SELF + aionotify.Flags.MOVE_SELF)
- await watcher.get_event()
- except OSError as e:
- logger.error(e)
- break
-
- await asyncio.sleep(0.1)
-
- new_sha = repo.refs[ref]
- if sha != new_sha:
- logger.info('Ref {} changed {} -> {}. Reloading.'.format(ref.decode(), sha.decode()[:6], new_sha.decode()[:6]))
- await load(app, ref=ref, **kwargs)
+ repo: Repo = app["trackers.data_repo"]
+
+ if hasattr(repo.refs, "refpath"):
+ sha: bytes
+ refnames, sha = repo.refs.follow(ref)
+ paths = [repo.refs.refpath(ref) for ref in refnames]
+ logger.debug(f"Watching paths {paths}")
+
+ with Inotify() as inotify:
+ for path in paths:
+ inotify.add_watch(
+ path.decode(),
+ Mask.MODIFY | Mask.DELETE_SELF | Mask.MOVE_SELF,
+ )
+ async for _ in inotify:
+ await asyncio.sleep(0.5)
+ new_sha: bytes = repo.refs[ref]
+ if sha != new_sha:
+ logger.info(
+ f"Ref {ref.decode()} changed "
+ f"{sha.decode()[:6]} -> {new_sha.decode()[:6]}. Reloading."
+ )
+ await load(app, ref=ref, **kwargs)
+ sha = new_sha
+ else:
+ logger.debug(f"Ref {ref.decode()} unchanged {sha.decode()[:6]}.")
else:
- logger.debug('No inotify reload on memory repo')
+ logger.debug("No inotify reload on memory repo")
except asyncio.CancelledError:
raise
except Exception:
- logger.exception('Error in load_with_watcher: ')
+ logger.exception("Error in load_with_inotify: ")
-async def load(app, ref=b'HEAD', **kwargs):
+async def load(app, ref=b"HEAD", **kwargs):
try:
- tree_reader = TreeReader(app['trackers.data_repo'], treeish=ref)
+ tree_reader = TreeReader(app["trackers.data_repo"], treeish=ref)
except KeyError:
pass
else:
try:
- app['config'] = yaml.safe_load(tree_reader.get('config.yaml').data)
+ app["config"] = yaml.safe_load(tree_reader.get("config.yaml").data)
except Exception:
- logger.exception('')
- app['config'] = {}
+ logger.exception("")
+ app["config"] = {}
await load_events(app, tree_reader, **kwargs)
-async def load_events(app, tree_reader, new_event_observable=Observable('new_event'), removed_event_observable=Observable('removed_event')):
- events = app['trackers.events']
- names = set(tree_reader.tree_items('events'))
+async def load_events(
+ app,
+ tree_reader,
+ new_event_observable=Observable("new_event"),
+ removed_event_observable=Observable("removed_event"),
+):
+ events = app["trackers.events"]
+ names = set(tree_reader.tree_items("events"))
for name in events.keys() - names:
await events[name].stop_and_complete_trackers()
await removed_event_observable(events.pop(name))
- load_event_fs = [load_event(name, app, events, tree_reader, new_event_observable) for name in names]
+ load_event_fs = [
+ asyncio.ensure_future(load_event(name, app, events, tree_reader, new_event_observable))
+ for name in names
+ ]
if load_event_fs:
await asyncio.wait(load_event_fs)
- logger.info('Events loaded.')
+ logger.info("Events loaded.")
async def load_event(name, app, events, tree_reader, new_event_observable):
@@ -101,39 +123,47 @@ async def load_event(name, app, events, tree_reader, new_event_observable):
events[name] = event = await Event.load(app, name, tree_reader)
await new_event_observable(event)
except yaml.YAMLError as e:
- logger.error(f'Error loading {name!r}: {e}')
+ logger.error(f"Error loading {name!r}: {e}")
except Exception:
- logger.exception(f'Error loading {name!r}: ')
+ logger.exception(f"Error loading {name!r}: ")
class Event(object):
def __init__(self, app, name, config=None, routes=None):
self.name = name
self.app = app
- self.logger = logging.getLogger(f'event.{name}')
+ self.logger = logging.getLogger(f"event.{name}")
self.trackers_started = False
self.starting_fut = None
self.batch_update_task = None
self.not_live_complete_trackers_task = None
- self.config_routes_change_observable = Observable(f'event.{name}.config_routes_change')
- self.rider_new_values_observable = Observable(f'event.{name}.rider_new_values')
- self.rider_pre_post_new_values_observable = Observable(f'event.{name}.rider_pre_post_new_values')
- self.rider_blocked_list_update_observable = Observable(f'event.{name}.rider_blocked_list_update')
- self.rider_off_route_blocked_list_update_observable = Observable(f'event.{name}.rider_off_route_blocked_list_update')
- self.rider_pre_post_blocked_list_update_observable = Observable(f'event.{name}.rider_pre_post_blocked_list_update')
- self.batch_update_observable = Observable(f'event.{name}.batch_update')
+ self.config_routes_change_observable = Observable(f"event.{name}.config_routes_change")
+ self.rider_new_values_observable = Observable(f"event.{name}.rider_new_values")
+ self.rider_pre_post_new_values_observable = Observable(
+ f"event.{name}.rider_pre_post_new_values"
+ )
+ self.rider_blocked_list_update_observable = Observable(
+ f"event.{name}.rider_blocked_list_update"
+ )
+ self.rider_off_route_blocked_list_update_observable = Observable(
+ f"event.{name}.rider_off_route_blocked_list_update"
+ )
+ self.rider_pre_post_blocked_list_update_observable = Observable(
+ f"event.{name}.rider_pre_post_blocked_list_update"
+ )
+ self.batch_update_observable = Observable(f"event.{name}.batch_update")
self.new_points = asyncio.Event()
- self.path = os.path.join('events', name)
+ self.path = os.path.join("events", name)
self.git_hash = None
- self.config_path = os.path.join(self.path, 'data.yaml')
+ self.config_path = os.path.join(self.path, "data.yaml")
self.config = config
- self.routes_msgpack_path = os.path.join(self.path, 'routes')
- self.routes_yaml_path = os.path.join(self.path, 'routes.yaml')
+ self.routes_msgpack_path = os.path.join(self.path, "routes")
+ self.routes_yaml_path = os.path.join(self.path, "routes.yaml")
self.routes = routes
self.routes_hash = hash_bytes(msgpack.dumps(routes)) if routes else None
@@ -147,7 +177,7 @@ async def load(cls, app, name, tree_reader):
async def reload(self, tree_reader):
_, git_hash = tree_reader.lookup(self.path)
if self.git_hash != git_hash:
- self.logger.info('Reloading')
+ self.logger.info("Reloading")
await self.stop_and_complete_trackers()
await self._load(tree_reader)
@@ -162,11 +192,11 @@ async def _load(self, tree_reader):
if tree_reader.exists(self.routes_yaml_path):
self.routes = yaml.safe_load(tree_reader.get(self.routes_yaml_path).data)
for route in self.routes:
- if route.get('data_hash'):
- route_data_path = os.path.join(self.path, 'routes_data', route.get('data_hash'))
+ if route.get("data_hash"):
+ route_data_path = os.path.join(self.path, "routes_data", route.get("data_hash"))
route_data = msgpack.loads(tree_reader.get(route_data_path).data, raw=False)
route.update(route_data)
- del route['data_hash']
+ del route["data_hash"]
self.routes_hash = hash_bytes(msgpack.dumps(self.routes))
elif tree_reader.exists(self.routes_msgpack_path):
routes_bytes = tree_reader.get(self.routes_msgpack_path).data
@@ -178,24 +208,38 @@ async def _load(self, tree_reader):
await self.config_routes_change_observable(self)
- async def save(self, message, author=None, tree_writer=None, save_routes=False, prevent_reload=True):
+ async def save(
+ self,
+ message,
+ author=None,
+ tree_writer=None,
+ save_routes=False,
+ prevent_reload=True,
+ ):
if tree_writer is None:
- tree_writer = TreeWriter(self.app['trackers.data_repo'])
- config_bytes = yaml.dump(self.config, default_flow_style=False, Dumper=YamlEventDumper).encode()
+ tree_writer = TreeWriter(self.app["trackers.data_repo"])
+ config_bytes = yaml.dump(
+ self.config, default_flow_style=False, Dumper=YamlEventDumper
+ ).encode()
tree_writer.set_data(self.config_path, config_bytes)
if save_routes:
if tree_writer.exists(self.routes_msgpack_path):
tree_writer.remove(self.routes_msgpack_path)
- routes_data_path = os.path.join(self.path, 'routes_data')
+ routes_data_path = os.path.join(self.path, "routes_data")
if tree_writer.exists(routes_data_path):
tree_writer.remove(routes_data_path)
routes = [copy.copy(route) for route in self.routes]
for route in routes:
route_data = {}
- for key in ('original_points', 'points', 'simplified_points_indexes', 'elevation'):
+ for key in (
+ "original_points",
+ "points",
+ "simplified_points_indexes",
+ "elevation",
+ ):
if key in route:
route_data[key] = route[key]
del route[key]
@@ -203,12 +247,18 @@ async def save(self, message, author=None, tree_writer=None, save_routes=False,
route_data_hash = hash_bytes(route_data_bytes)
route_data_path = os.path.join(routes_data_path, route_data_hash)
tree_writer.set_data(route_data_path, route_data_bytes)
- route['data_hash'] = route_data_hash
+ route["data_hash"] = route_data_hash
- routes_bytes = yaml.dump(routes, default_flow_style=False, Dumper=YamlEventDumper).encode()
+ routes_bytes = yaml.dump(
+ routes, default_flow_style=False, Dumper=YamlEventDumper
+ ).encode()
tree_writer.set_data(self.routes_yaml_path, routes_bytes)
- tree_writer.commit(message, author=author)
+ if isinstance(message, str):
+ message = message.encode()
+ if isinstance(author, str):
+ author = author.encode()
+ tree_writer.do_commit(message, author=author)
if prevent_reload:
_, self.git_hash = tree_writer.lookup(self.path)
else:
@@ -216,10 +266,7 @@ async def save(self, message, author=None, tree_writer=None, save_routes=False,
@property
def admin_allowed_principals(self):
- return tuple(chain(
- self.app['config'].get('admin', ()),
- self.config.get('admin', ())
- ))
+ return tuple(chain(self.app["config"].get("admin", ()), self.config.get("admin", ())))
async def start_trackers(self, analyse=True):
self.start_trackers_without_wait(analyse)
@@ -235,12 +282,12 @@ def _start_done(self, fut):
self.starting_fut = None
async def _start_trackers(self, analyse):
- self.logger.info('Starting.')
+ self.logger.info("Starting.")
# analyse = self.config.get('analyse', False)
- replay = self.config.get('replay', False)
- is_live = self.config.get('live', False)
- self.event_start = self.config.get('event_start')
+ replay = self.config.get("replay", False)
+ is_live = self.config.get("live", False)
+ self.event_start = self.config.get("event_start")
self.riders_objects = {}
self.riders_current_values = {}
@@ -250,123 +297,178 @@ async def _start_trackers(self, analyse):
self.riders_off_route_updated = set()
self.riders_pre_post_updated = set()
- tree_reader = TreeReader(self.app['trackers.data_repo'], treeish=self.git_hash) if self.git_hash else None
- has_static = tree_reader.exists(os.path.join('static')) if tree_reader else False
- has_static_analyse = has_static and self.config.get('static_analyse', False) and not replay
+ tree_reader = (
+ TreeReader(self.app["trackers.data_repo"], treeish=self.git_hash)
+ if self.git_hash
+ else None
+ )
+ has_static = tree_reader.exists(os.path.join("static")) if tree_reader else False
+ has_static_analyse = has_static and self.config.get("static_analyse", False) and not replay
if analyse and not has_static_analyse:
loop = asyncio.get_event_loop()
analyse_routes = await loop.run_in_executor(None, get_analyse_routes, self.routes)
- find_closest_cache_dir = os.path.join(self.app['trackers.settings']['cache_path'], 'find_closest')
+ find_closest_cache_dir = os.path.join(
+ self.app["trackers.settings"]["cache_path"], "find_closest"
+ )
os.makedirs(find_closest_cache_dir, exist_ok=True)
if self.routes:
- find_closest_cache = PersistedFuncCache(os.path.join(find_closest_cache_dir, f'2-{self.routes_hash}'))
- logger.info(f'find_closest_cache: {find_closest_cache.path}')
+ find_closest_cache = PersistedFuncCache(
+ os.path.join(find_closest_cache_dir, f"2-{self.routes_hash}")
+ )
+ logger.info(f"find_closest_cache: {find_closest_cache.path}")
else:
find_closest_cache = None
if replay:
replay_config = replay if isinstance(replay, dict) else {}
replay_kwargs = {
- 'replay_start': datetime.now(),
- 'speed_multiply': replay_config.get('speed_multiply', 2),
- 'offset': timedelta(**replay_config.get('offset', {})),
- 'event_start_time': self.event_start,
+ "replay_start": datetime.now(),
+ "speed_multiply": replay_config.get("speed_multiply", 2),
+ "offset": timedelta(**replay_config.get("offset", {})),
+ "event_start_time": self.event_start,
}
- self.event_start = replay_kwargs['replay_start'] + replay_kwargs['offset']
+ self.event_start = replay_kwargs["replay_start"] + replay_kwargs["offset"]
- for rider in self.config['riders']:
- rider_name = rider['name']
+ for rider in self.config["riders"]:
+ rider_name = rider["name"]
self.riders_objects[rider_name] = objects = RiderObjects(rider_name, self)
- objects.data_tracker = await DataTracker.start(rider, self.config.get('tracker_end'))
+ objects.data_tracker = await DataTracker.start(rider, self.config.get("tracker_end"))
if has_static:
- for rider in self.config['riders']:
- rider_name = rider['name']
- tracker = await start_implicit_static_event_tracker(self, rider_name, 'source', tree_reader)
+ for rider in self.config["riders"]:
+ rider_name = rider["name"]
+ tracker = await start_implicit_static_event_tracker(
+ self, rider_name, "source", tree_reader
+ )
self.riders_objects[rider_name].source_trackers.append(tracker)
else:
rider_tracker_start_fs = defaultdict(list)
- for rider in self.config['riders']:
- rider_name = rider['name']
- rider_trackers = tuple(chain(
- (rider['tracker'], ) if 'tracker' in rider and rider['tracker'] else (),
- rider.get('trackers', ())
- ))
+ for rider in self.config["riders"]:
+ rider_name = rider["name"]
+ rider_trackers = tuple(
+ chain(
+ (rider["tracker"],) if "tracker" in rider and rider["tracker"] else (),
+ rider.get("trackers", ()),
+ )
+ )
for tracker in rider_trackers:
- start_tracker = self.app['start_event_trackers'][tracker['type']]
- start = tracker.get('start') or self.config.get('tracker_start')
- end = tracker.get('end') or self.config.get('tracker_end')
- start_fut = asyncio.ensure_future(start_tracker(self.app, self, rider_name, tracker, start, end))
+ start_tracker = self.app["start_event_trackers"][tracker["type"]]
+ start = tracker.get("start") or self.config.get("tracker_start")
+ end = tracker.get("end") or self.config.get("tracker_end")
+ start_fut = asyncio.ensure_future(
+ start_tracker(self.app, self, rider_name, tracker, start, end)
+ )
rider_tracker_start_fs[rider_name].append(start_fut)
all_start_fs = list(chain.from_iterable(rider_tracker_start_fs.values()))
if all_start_fs:
await asyncio.wait(all_start_fs)
- for rider in self.config['riders']:
- for start_fut in rider_tracker_start_fs[rider['name']]:
+ for rider in self.config["riders"]:
+ for start_fut in rider_tracker_start_fs[rider["name"]]:
tracker = start_fut.result()
- self.riders_objects[rider['name']].source_trackers.append(tracker)
+ self.riders_objects[rider["name"]].source_trackers.append(tracker)
- for rider in self.config['riders']:
- rider_name = rider['name']
+ for rider in self.config["riders"]:
+ rider_name = rider["name"]
objects = self.riders_objects[rider_name]
# Should these not be a part of objects?
- self.riders_current_values[rider['name']] = {}
- self.riders_pre_post_values[rider['name']] = {}
+ self.riders_current_values[rider["name"]] = {}
+ self.riders_pre_post_values[rider["name"]] = {}
- objects.combined_tracker = tracker = await Combined.start(f'combined.{rider_name}', tuple(chain((objects.data_tracker, ), objects.source_trackers)))
+ objects.combined_tracker = tracker = await Combined.start(
+ f"combined.{rider_name}",
+ tuple(chain((objects.data_tracker,), objects.source_trackers)),
+ )
if replay:
tracker = await start_replay_tracker(tracker, **replay_kwargs)
- if analyse and rider.get('type', 'rider') == 'rider':
+ if analyse and rider.get("type", "rider") == "rider":
if has_static_analyse:
objects.analyse_tracker = tracker = await start_implicit_static_event_tracker(
- self, rider_name, 'analyse', tree_reader)
+ self, rider_name, "analyse", tree_reader
+ )
objects.off_route_tracker = await start_implicit_static_event_tracker(
- self, rider_name, 'off_route', tree_reader)
+ self, rider_name, "off_route", tree_reader
+ )
# As we intentionally don't store pre_post, just create a blank tracker.
- objects.pre_post_tracker = Tracker('null')
+ objects.pre_post_tracker = Tracker("null")
else:
objects.analyse_tracker = tracker = await AnalyseTracker.start(
- tracker, self.event_start, analyse_routes, find_closest_cache=find_closest_cache,
- processing_lock=self.app['analyse_processing_lock'])
- objects.off_route_tracker = await index_and_hash_tracker(tracker.off_route_tracker)
- objects.pre_post_tracker = await index_and_hash_tracker(tracker.pre_post_tracker)
- await self.on_rider_pre_post_new_points(rider['name'], objects.pre_post_tracker, objects.pre_post_tracker.points)
- objects.pre_post_tracker.new_points_observable.subscribe(partial(self.on_rider_pre_post_new_points, rider['name']))
- tracker.not_pre_post_observable.subscribe(partial(self.on_rider_not_pre_post, rider['name']))
+ tracker,
+ self.event_start,
+ analyse_routes,
+ find_closest_cache=find_closest_cache,
+ processing_lock=self.app["analyse_processing_lock"],
+ )
+ objects.off_route_tracker = await index_and_hash_tracker(
+ tracker.off_route_tracker
+ )
+ objects.pre_post_tracker = await index_and_hash_tracker(
+ tracker.pre_post_tracker
+ )
+ await self.on_rider_pre_post_new_points(
+ rider["name"],
+ objects.pre_post_tracker,
+ objects.pre_post_tracker.points,
+ )
+ objects.pre_post_tracker.new_points_observable.subscribe(
+ partial(self.on_rider_pre_post_new_points, rider["name"])
+ )
+ tracker.not_pre_post_observable.subscribe(
+ partial(self.on_rider_not_pre_post, rider["name"])
+ )
objects.off_route_blocked_list = BlockedList.from_tracker(
- objects.off_route_tracker, entire_block=not is_live,
- new_update_callbacks=(partial(self.on_rider_off_route_update, rider['name']), ))
+ objects.off_route_tracker,
+ entire_block=not is_live,
+ new_update_callbacks=(partial(self.on_rider_off_route_update, rider["name"]),),
+ )
objects.pre_post_blocked_list = BlockedList.from_tracker(
- objects.pre_post_tracker, entire_block=not is_live,
- new_update_callbacks=(partial(self.rider_pre_post_blocked_list_update_observable, self, rider['name']), ))
+ objects.pre_post_tracker,
+ entire_block=not is_live,
+ new_update_callbacks=(
+ partial(
+ self.rider_pre_post_blocked_list_update_observable,
+ self,
+ rider["name"],
+ ),
+ ),
+ )
tracker = await index_and_hash_tracker(tracker)
- await self.on_rider_new_points(rider['name'], tracker, tracker.points)
- tracker.new_points_observable.subscribe(partial(self.on_rider_new_points, rider['name']))
- tracker.reset_points_observable.subscribe(partial(self.on_rider_reset_points, rider['name']))
+ await self.on_rider_new_points(rider["name"], tracker, tracker.points)
+ tracker.new_points_observable.subscribe(
+ partial(self.on_rider_new_points, rider["name"])
+ )
+ tracker.reset_points_observable.subscribe(
+ partial(self.on_rider_reset_points, rider["name"])
+ )
objects.tracker = tracker
objects.blocked_list = BlockedList.from_tracker(
- tracker, entire_block=not is_live,
- new_update_callbacks=(partial(self.rider_blocked_list_update_observable, self, rider['name']), ))
+ tracker,
+ entire_block=not is_live,
+ new_update_callbacks=(
+ partial(self.rider_blocked_list_update_observable, self, rider["name"]),
+ ),
+ )
if is_live:
self.batch_update_task = asyncio.ensure_future(self.batch_update_loop())
else:
- self.not_live_complete_trackers_task = asyncio.ensure_future(self.not_live_complete_trackers())
+ self.not_live_complete_trackers_task = asyncio.ensure_future(
+ self.not_live_complete_trackers()
+ )
self.not_live_complete_trackers_task.add_done_callback(general_fut_done_callback)
self.trackers_started = True
- self.logger.info('Started.')
+ self.logger.info("Started.")
async def stop_and_complete_trackers(self):
if self.starting_fut:
@@ -388,7 +490,7 @@ async def stop_and_complete_trackers(self):
except asyncio.CancelledError:
pass
except Exception:
- self.logger.exception('Unhandled tracker error: ')
+ self.logger.exception("Unhandled tracker error: ")
if self.not_live_complete_trackers_task:
await self.not_live_complete_trackers_task
@@ -402,7 +504,9 @@ async def stop_and_complete_trackers(self):
self.trackers_started = False
async def not_live_complete_trackers(self):
- await asyncio.wait([rider_objs.tracker.completed for rider_objs in self.riders_objects.values()])
+ await asyncio.wait(
+ [rider_objs.tracker.completed for rider_objs in self.riders_objects.values()]
+ )
riders = set(self.riders_objects.keys())
await self.batch_update(riders, riders, riders)
@@ -411,8 +515,8 @@ async def on_rider_new_points(self, rider_name, tracker, new_points):
values = self.riders_current_values[rider_name]
for point in new_points:
values.update(point)
- if 'position' in point:
- values['position_time'] = point['time']
+ if "position" in point:
+ values["position_time"] = point["time"]
# if 'rider_status' in values:
# with suppress(KeyError):
# del values['position']
@@ -423,9 +527,11 @@ async def on_rider_new_points(self, rider_name, tracker, new_points):
async def on_rider_off_route_update(self, rider_name, blocked_list, update):
self.riders_off_route_updated.add(rider_name)
self.new_points.set()
- await self.rider_off_route_blocked_list_update_observable(self, rider_name, blocked_list, update)
+ await self.rider_off_route_blocked_list_update_observable(
+ self, rider_name, blocked_list, update
+ )
- pre_post_update_main_keys = {'time', 'battery', 'tk_status', 'tk_config'}
+ pre_post_update_main_keys = {"time", "battery", "tk_status", "tk_config"}
async def on_rider_pre_post_new_points(self, rider_name, tracker, new_points):
if new_points:
@@ -437,18 +543,20 @@ async def on_rider_pre_post_new_points(self, rider_name, tracker, new_points):
pre_post_values.update(point)
# TODO, rather than do this check, we should make sure the values come in in the right order.
- point_is_newer = point['time'] > values.get('time', min_time)
+ point_is_newer = point["time"] > values.get("time", min_time)
if point_is_newer:
- has_values_update, values_update = spy(((k, v) for k, v in point.items() if k in self.pre_post_update_main_keys))
+ has_values_update, values_update = spy(
+ ((k, v) for k, v in point.items() if k in self.pre_post_update_main_keys)
+ )
if has_values_update:
values_update = list(values_update)
values.update(values_update)
values_updated = True
- if 'position' in point:
- pre_post_values['position_time'] = point['time']
+ if "position" in point:
+ pre_post_values["position_time"] = point["time"]
if point_is_newer:
- values['position_time'] = point['time']
+ values["position_time"] = point["time"]
values_updated = True
if values_updated:
@@ -481,25 +589,34 @@ async def on_rider_reset_points(self, rider_name, tracker):
def rider_sort_key_func(self, riders_predicted_points, rider_name):
rider_values = self.riders_current_values.get(rider_name, {})
- finished = 'finished_time' in rider_values
- time_to_finish = rider_values['finished_time'] - self.event_start if finished else None
- has_dist_on_route = 'dist_route' in rider_values
- dist_on_route = riders_predicted_points.get(rider_name, {}).get('dist_route') or rider_values.get('dist_route', 0)
+ finished = "finished_time" in rider_values
+ time_to_finish = rider_values["finished_time"] - self.event_start if finished else None
+ has_dist_on_route = "dist_route" in rider_values
+ dist_on_route = riders_predicted_points.get(rider_name, {}).get(
+ "dist_route"
+ ) or rider_values.get("dist_route", 0)
return not finished, time_to_finish, not has_dist_on_route, 0 - dist_on_route
async def batch_update_loop(self):
- batch_update_interval = self.config.get('batch_update_interval') or 2
- predicted_update_interval = self.config.get('predicted_update_interval') or 10
+ batch_update_interval = self.config.get("batch_update_interval") or 2
+ predicted_update_interval = self.config.get("predicted_update_interval") or 10
while True:
await asyncio.sleep(batch_update_interval)
with suppress(asyncio.TimeoutError):
- await asyncio.wait_for(self.new_points.wait(), predicted_update_interval - batch_update_interval)
+ await asyncio.wait_for(
+ self.new_points.wait(),
+ predicted_update_interval - batch_update_interval,
+ )
try:
- await self.batch_update(self.riders_updated, self.riders_off_route_updated, self.riders_pre_post_updated)
+ await self.batch_update(
+ self.riders_updated,
+ self.riders_off_route_updated,
+ self.riders_pre_post_updated,
+ )
except asyncio.CancelledError:
raise
except Exception:
- self.logger.exception('Error in batch_update:')
+ self.logger.exception("Error in batch_update:")
self.riders_updated = set()
self.riders_off_route_updated = set()
self.riders_pre_post_updated = set()
@@ -510,8 +627,11 @@ async def batch_update(self, riders_updated, riders_off_route_updated, riders_pr
return
time = datetime.now()
riders_predicted_points = {
- rider_objects.rider_name: rider_objects.analyse_tracker.get_predicted_position(time) or {}
- for rider_objects in self.riders_objects.values() if isinstance(rider_objects.analyse_tracker, AnalyseTracker)}
+ rider_objects.rider_name: rider_objects.analyse_tracker.get_predicted_position(time)
+ or {}
+ for rider_objects in self.riders_objects.values()
+ if isinstance(rider_objects.analyse_tracker, AnalyseTracker)
+ }
sort_key_func = partial(self.rider_sort_key_func, riders_predicted_points)
rider_names_sorted = list(sorted(self.riders_objects.keys(), key=sort_key_func))
@@ -522,47 +642,60 @@ async def batch_update(self, riders_updated, riders_off_route_updated, riders_pr
last_point = None
if leader_objects.analyse_tracker:
for point in leader_objects.analyse_tracker.points:
- if 'dist_route' in point:
- going_forward = point['dist_route'] > last_point['dist_route'] if last_point else True
+ if "dist_route" in point:
+ going_forward = (
+ point["dist_route"] > last_point["dist_route"] if last_point else True
+ )
if going_forward:
- leader_points.append((point['dist_route'], point['time']))
+ leader_points.append((point["dist_route"], point["time"]))
last_point = point
- if 'dist_route' in riders_predicted_points.get(leader, ()):
- leader_points.append((riders_predicted_points[leader]['dist_route'], time))
+ if "dist_route" in riders_predicted_points.get(leader, ()):
+ leader_points.append((riders_predicted_points[leader]["dist_route"], time))
if leader_points:
- self.riders_current_values[leader]['leader_time_diff'] = timedelta(0)
+ self.riders_current_values[leader]["leader_time_diff"] = timedelta(0)
for rider_name in rider_names_sorted[1:]:
- leader_time_diff = self.get_leader_time_diff(rider_name, riders_predicted_points,
- leader, leader_points, time)
+ leader_time_diff = self.get_leader_time_diff(
+ rider_name, riders_predicted_points, leader, leader_points, time
+ )
if leader_time_diff is not None:
if rider_name in riders_predicted_points:
- riders_predicted_points[rider_name]['leader_time_diff'] = leader_time_diff
- self.riders_current_values[rider_name]['leader_time_diff'] = leader_time_diff
+ riders_predicted_points[rider_name]["leader_time_diff"] = leader_time_diff
+ self.riders_current_values[rider_name]["leader_time_diff"] = leader_time_diff
else:
with suppress(KeyError):
- del self.riders_current_values[rider_name]['leader_time_diff']
-
- self.riders_predicted_points = {key: value for key, value in riders_predicted_points.items() if value}
- await self.batch_update_observable(self, time, riders_updated, riders_off_route_updated, riders_pre_post_updated)
-
- def get_leader_time_diff(self, rider_name, riders_predicted_points, leader, leader_points, time):
+ del self.riders_current_values[rider_name]["leader_time_diff"]
+
+ self.riders_predicted_points = {
+ key: value for key, value in riders_predicted_points.items() if value
+ }
+ await self.batch_update_observable(
+ self,
+ time,
+ riders_updated,
+ riders_off_route_updated,
+ riders_pre_post_updated,
+ )
+
+ def get_leader_time_diff(
+ self, rider_name, riders_predicted_points, leader, leader_points, time
+ ):
rider_predicted_points = riders_predicted_points.get(rider_name)
rider_values = self.riders_current_values.get(rider_name)
- if rider_values and 'finished_time' in rider_values:
+ if rider_values and "finished_time" in rider_values:
leader_values = self.riders_current_values.get(leader)
- if 'finished_time' in leader_values:
- return rider_values['finished_time'] - leader_values['finished_time']
+ if "finished_time" in leader_values:
+ return rider_values["finished_time"] - leader_values["finished_time"]
- if rider_values and 'position_time' in rider_values:
+ if rider_values and "position_time" in rider_values:
rider_dist_route = None
rider_time = None
- if rider_predicted_points and 'dist_route' in rider_predicted_points:
- rider_dist_route = rider_predicted_points['dist_route']
- rider_time = rider_predicted_points['time']
- elif rider_values and 'dist_route' in rider_values:
- rider_dist_route = rider_values['dist_route']
- rider_time = rider_values['position_time']
+ if rider_predicted_points and "dist_route" in rider_predicted_points:
+ rider_dist_route = rider_predicted_points["dist_route"]
+ rider_time = rider_predicted_points["time"]
+ elif rider_values and "dist_route" in rider_values:
+ rider_dist_route = rider_values["dist_route"]
+ rider_time = rider_values["position_time"]
if rider_dist_route:
i = bisect(leader_points, (rider_dist_route,))
if i < len(leader_points):
@@ -582,61 +715,67 @@ def get_leader_time_diff(self, rider_name, riders_predicted_points, leader, lead
async def convert_to_static(self, tree_writer):
try:
await self.start_trackers(analyse=False)
- for rider in self.config['riders']:
- rider_name = rider['name']
- tracker = await Combined.start(rider_name, self.riders_objects[rider_name].source_trackers)
+ for rider in self.config["riders"]:
+ rider_name = rider["name"]
+ tracker = await Combined.start(
+ rider_name, self.riders_objects[rider_name].source_trackers
+ )
await tracker.complete()
with suppress(KeyError):
- del rider['trackers']
+ del rider["trackers"]
with suppress(KeyError):
- del rider['tracker']
+ del rider["tracker"]
# Remove old static file.
with suppress(KeyError):
- tree_writer.remove(os.path.join('events', self.name, rider_name))
+ tree_writer.remove(os.path.join("events", self.name, rider_name))
- path = os.path.join('events', self.name, 'static', rider_name, 'source')
+ path = os.path.join("events", self.name, "static", rider_name, "source")
tree_writer.set_data(path, msgpack.dumps(tracker.points, default=json_encode))
- self.config['live'] = False
- await self.save(f'{self.name}: convert_to_static', tree_writer=tree_writer)
+ self.config["live"] = False
+ await self.save(f"{self.name}: convert_to_static", tree_writer=tree_writer)
finally:
await self.stop_and_complete_trackers()
async def store_analyse(self, tree_writer):
- assert tree_writer.exists(os.path.join('events', self.name, 'static'))
+ assert tree_writer.exists(os.path.join("events", self.name, "static"))
try:
- self.config['static_analyse'] = False
+ self.config["static_analyse"] = False
await self.start_trackers(analyse=True)
- for rider in self.config['riders']:
- rider_name = rider['name']
+ for rider in self.config["riders"]:
+ rider_name = rider["name"]
trackers = (
- (self.riders_objects[rider_name].analyse_tracker, 'analyse'),
- (self.riders_objects[rider_name].off_route_tracker, 'off_route'),
+ (self.riders_objects[rider_name].analyse_tracker, "analyse"),
+ (self.riders_objects[rider_name].off_route_tracker, "off_route"),
)
for tracker, sub_tracker_type in trackers:
if tracker:
await tracker.complete()
- path = os.path.join('events', self.name, 'static', rider_name, sub_tracker_type)
- tree_writer.set_data(path, msgpack.dumps(tracker.points, default=json_encode))
-
- self.config['static_analyse'] = True
- await self.save(f'{self.name}: store static analyse', tree_writer=tree_writer)
+ path = os.path.join(
+ "events", self.name, "static", rider_name, sub_tracker_type
+ )
+ tree_writer.set_data(
+ path, msgpack.dumps(tracker.points, default=json_encode)
+ )
+
+ self.config["static_analyse"] = True
+ await self.save(f"{self.name}: store static analyse", tree_writer=tree_writer)
finally:
await self.stop_and_complete_trackers()
async def start_implicit_static_event_tracker(event, rider_name, sub_type, tree_reader):
- tracker = Tracker(f'static.{rider_name}.{sub_type}')
- path = os.path.join('static', rider_name, sub_type)
+ tracker = Tracker(f"static.{rider_name}.{sub_type}")
+ path = os.path.join("static", rider_name, sub_type)
points = msgpack.loads(tree_reader.get(path).data, raw=False)
for point in points:
- if 'time' in point:
- point['time'] = datetime.fromtimestamp(point['time'])
- if 'server_time' in point:
- point['server_time'] = datetime.fromtimestamp(point['server_time'])
- if 'finished_time' in point:
- point['finished_time'] = datetime.fromtimestamp(point['finished_time'])
+ if "time" in point:
+ point["time"] = datetime.fromtimestamp(point["time"])
+ if "server_time" in point:
+ point["server_time"] = datetime.fromtimestamp(point["server_time"])
+ if "finished_time" in point:
+ point["finished_time"] = datetime.fromtimestamp(point["finished_time"])
# print(event.name, rider_name, sub_type, len(points))
await tracker.new_points(points)
@@ -646,22 +785,20 @@ async def start_implicit_static_event_tracker(event, rider_name, sub_type, tree_
dict_key_order = {
# Event
- 'title': -10,
- 'event_start': -9,
- 'tracker_start': -8,
- 'tracker_end': -7,
- 'live': -6,
- 'analyse': -5,
- 'replay': -4,
- 'time_show_days': -3,
- 'riders': 10,
-
+ "title": -10,
+ "event_start": -9,
+ "tracker_start": -8,
+ "tracker_end": -7,
+ "live": -6,
+ "analyse": -5,
+ "replay": -4,
+ "time_show_days": -3,
+ "riders": 10,
# Rider
- 'name': -5,
- 'name_short': -4,
-
+ "name": -5,
+ "name_short": -4,
# Tracker
- 'type': -10,
+ "type": -10,
}
@@ -674,7 +811,7 @@ def dict_key_order_key(item):
def yaml_represent_dict(self, dict):
# Control the order of the key
mapping = sorted(dict.items(), key=dict_key_order_key)
- return self.represent_mapping('tag:yaml.org,2002:map', mapping)
+ return self.represent_mapping("tag:yaml.org,2002:map", mapping)
class YamlEventDumper(yaml.Dumper):
@@ -702,12 +839,11 @@ class RiderObjects(object):
class DataTracker(Tracker):
-
@classmethod
async def start(cls, rider_data, end):
self = cls(f'data.{rider_data["name"]}')
self.rider_data = rider_data
- await self.new_points(rider_data.get('points', ()))
+ await self.new_points(rider_data.get("points", ()))
if end:
now = datetime.now()
@@ -720,5 +856,5 @@ async def start(cls, rider_data, end):
async def add_points(self, new_points):
await self.new_points(new_points)
- self.rider_data['points'] = self.points
+ self.rider_data["points"] = self.points
# Caller is responsible to call event.save
diff --git a/trackers/general.py b/trackers/general.py
index c177b5f..ed61070 100644
--- a/trackers/general.py
+++ b/trackers/general.py
@@ -9,9 +9,9 @@
from datetime import datetime, timedelta
import msgpack
+from dulwich_tree import TreeReader
from trackers.base import Tracker
-from trackers.dulwich_helpers import TreeReader
def json_encode(obj):
@@ -27,36 +27,52 @@ def json_encode(obj):
async def static_start_event_tracker(app, event, rider_name, tracker_data, start, end):
- tracker = Tracker('static.{}'.format(tracker_data['name']))
- path = os.path.join('events', event.name, tracker_data['name'])
- data = TreeReader(app['trackers.data_repo']).get(path).data
+ tracker = Tracker("static.{}".format(tracker_data["name"]))
+ path = os.path.join("events", event.name, tracker_data["name"])
+ data = TreeReader(app["trackers.data_repo"]).get(path).data
points = {
- 'json': lambda data: json.loads(data.decode()),
- 'msgpack': lambda data: msgpack.loads(data, raw=False)
- }[tracker_data.get('format', 'json')](data)
+ "json": lambda data: json.loads(data.decode()),
+ "msgpack": lambda data: msgpack.loads(data, raw=False),
+ }[tracker_data.get("format", "json")](data)
for point in points:
- if 'time' in point:
- point['time'] = datetime.fromtimestamp(point['time'])
- if 'server_time' in point:
- point['server_time'] = datetime.fromtimestamp(point['server_time'])
- if 'finished_time' in point:
- point['finished_time'] = datetime.fromtimestamp(point['finished_time'])
+ if "time" in point:
+ point["time"] = datetime.fromtimestamp(point["time"])
+ if "server_time" in point:
+ point["server_time"] = datetime.fromtimestamp(point["server_time"])
+ if "finished_time" in point:
+ point["finished_time"] = datetime.fromtimestamp(point["finished_time"])
await tracker.new_points(points)
tracker.completed.set_result(None)
return tracker
-async def start_replay_tracker(org_tracker, event_start_time, replay_start, offset=timedelta(0), speed_multiply=2000):
- replay_tracker = Tracker('replay.{}'.format(org_tracker.name))
+async def start_replay_tracker(
+ org_tracker,
+ event_start_time,
+ replay_start,
+ offset=timedelta(0),
+ speed_multiply=2000,
+):
+ replay_tracker = Tracker("replay.{}".format(org_tracker.name))
replay_task = asyncio.ensure_future(
- replay(replay_tracker, org_tracker, event_start_time, replay_start, offset, speed_multiply))
+ replay(
+ replay_tracker,
+ org_tracker,
+ event_start_time,
+ replay_start,
+ offset,
+ speed_multiply,
+ )
+ )
replay_tracker.stop = replay_task.cancel
replay_tracker.completed = replay_task
return replay_tracker
-async def replay(replay_tracker, org_tracker, event_start_time, replay_start, offset, speed_multiply):
+async def replay(
+ replay_tracker, org_tracker, event_start_time, replay_start, offset, speed_multiply
+):
point_i = 0
while not org_tracker.completed.done() or point_i < len(org_tracker.points):
now = datetime.now()
@@ -65,53 +81,71 @@ async def replay(replay_tracker, org_tracker, event_start_time, replay_start, of
while point_i < len(org_tracker.points):
try:
point = org_tracker.points[point_i]
- time = point.get('time') or point.get('server_time')
+ time = point.get("time") or point.get("server_time")
new_time = replay_start + ((time - event_start_time + offset) / speed_multiply)
if new_time <= now:
point_i += 1
- point['time'] = new_time
+ point["time"] = new_time
new_points.append(point)
else:
break
except Exception:
- replay_tracker.logger.exception('Error in replay:')
+ replay_tracker.logger.exception("Error in replay:")
if new_points:
await replay_tracker.new_points(new_points)
if new_time:
delay = (new_time - now).total_seconds()
- replay_tracker.logger.debug(f'Sleep for {delay} sec. till {new_time}')
+ replay_tracker.logger.debug(f"Sleep for {delay} sec. till {new_time}")
await asyncio.sleep(delay)
else:
- replay_tracker.logger.debug('Sleep 1')
+ replay_tracker.logger.debug("Sleep 1")
await asyncio.sleep(1)
-async def wrapped_tracker_start_event(start_wraped, app, event, rider_name, tracker_data, start, end):
- start_tracker = app['start_event_trackers'][tracker_data['tracker']['type']]
- org_tracker = await start_tracker(app, event, rider_name, tracker_data['tracker'], start, end)
+async def wrapped_tracker_start_event(
+ start_wraped, app, event, rider_name, tracker_data, start, end
+):
+ start_tracker = app["start_event_trackers"][tracker_data["tracker"]["type"]]
+ org_tracker = await start_tracker(app, event, rider_name, tracker_data["tracker"], start, end)
return await start_wraped(org_tracker, tracker_data)
async def cropped_tracker_start(org_tracker, tracker_data):
- cropped_tracker = Tracker('cropped.{}'.format(org_tracker.name), org_tracker.completed)
+ cropped_tracker = Tracker("cropped.{}".format(org_tracker.name), org_tracker.completed)
cropped_tracker.stop = org_tracker.stop
cropped_tracker.org_tracker = org_tracker
- await cropped_tracker_newpoints(cropped_tracker, tracker_data.get('start'), tracker_data.get('end'), org_tracker, org_tracker.points)
+ await cropped_tracker_newpoints(
+ cropped_tracker,
+ tracker_data.get("start"),
+ tracker_data.get("end"),
+ org_tracker,
+ org_tracker.points,
+ )
org_tracker.new_points_observable.subscribe(
- functools.partial(cropped_tracker_newpoints, cropped_tracker, tracker_data.get('start'), tracker_data.get('end')))
+ functools.partial(
+ cropped_tracker_newpoints,
+ cropped_tracker,
+ tracker_data.get("start"),
+ tracker_data.get("end"),
+ )
+ )
return cropped_tracker
async def cropped_tracker_newpoints(cropped_tracker, start, end, org_tracker, new_points):
- points = [point for point in new_points if (not end or point['time'] < end) and (not start or point['time'] > start)]
+ points = [
+ point
+ for point in new_points
+ if (not end or point["time"] < end) and (not start or point["time"] > start)
+ ]
if points:
await cropped_tracker.new_points(points)
async def index_and_hash_tracker(org_tracker, hasher=None):
- ih_tracker = Tracker('indexed_and_hashed.{}'.format(org_tracker.name), org_tracker.completed)
+ ih_tracker = Tracker("indexed_and_hashed.{}".format(org_tracker.name), org_tracker.completed)
ih_tracker.stop = org_tracker.stop
ih_tracker.org_tracker = org_tracker
if hasher is None:
@@ -120,9 +154,11 @@ async def index_and_hash_tracker(org_tracker, hasher=None):
await index_and_hash_tracker_org_newpoints(ih_tracker, org_tracker, org_tracker.points)
org_tracker.new_points_observable.subscribe(
- functools.partial(index_and_hash_tracker_org_newpoints, ih_tracker))
+ functools.partial(index_and_hash_tracker_org_newpoints, ih_tracker)
+ )
org_tracker.reset_points_observable.subscribe(
- functools.partial(index_and_hash_tracker_org_reset_points, ih_tracker))
+ functools.partial(index_and_hash_tracker_org_reset_points, ih_tracker)
+ )
return ih_tracker
@@ -130,9 +166,9 @@ async def index_and_hash_tracker(org_tracker, hasher=None):
def index_and_hash_list(points, start, hasher):
ih_points = [copy(point) for point in points]
for i, (ih_point, org_point) in enumerate(zip(ih_points, points), start=start):
- ih_point['index'] = i
+ ih_point["index"] = i
hasher.update(json_dumps(org_point).encode())
- ih_point['hash'] = urlsafe_b64encode(hasher.digest()[:3]).decode('ascii')
+ ih_point["hash"] = urlsafe_b64encode(hasher.digest()[:3]).decode("ascii")
return ih_points
@@ -146,26 +182,29 @@ async def index_and_hash_tracker_org_reset_points(ih_tracker, org_tracker):
async def filter_inaccurate_tracker_start(org_tracker, tracker_data):
- filtered_tracker = Tracker('filter_inaccurate.{}'.format(org_tracker.name), org_tracker.completed)
+ filtered_tracker = Tracker(
+ "filter_inaccurate.{}".format(org_tracker.name), org_tracker.completed
+ )
filtered_tracker.stop = org_tracker.stop
filtered_tracker.org_tracker = org_tracker
await filter_inaccurate_tracker_newpoints(filtered_tracker, org_tracker, org_tracker.points)
org_tracker.new_points_observable.subscribe(
- functools.partial(filter_inaccurate_tracker_newpoints, filtered_tracker))
+ functools.partial(filter_inaccurate_tracker_newpoints, filtered_tracker)
+ )
return filtered_tracker
async def filter_inaccurate_tracker_newpoints(filtered_tracker, org_tracker, new_points):
points = []
for point in new_points:
- if point.get('accuracy', 0) >= 500:
+ if point.get("accuracy", 0) >= 500:
point = copy(point)
- del point['position']
+ del point["position"]
points.append(point)
if points:
await filtered_tracker.new_points(points)
def hash_bytes(b):
- return urlsafe_b64encode(hashlib.sha1(b).digest()).decode('ascii')
+ return urlsafe_b64encode(hashlib.sha1(b).digest()).decode("ascii")
diff --git a/trackers/modules.py b/trackers/modules.py
index 7f0a099..ee12611 100644
--- a/trackers/modules.py
+++ b/trackers/modules.py
@@ -2,12 +2,12 @@
async def config_modules(app, settings):
- import trackers.sources.traccar
+ import trackers.sources.garmin_inreach
+ import trackers.sources.matrix
import trackers.sources.spot
import trackers.sources.tkstorage
+ import trackers.sources.traccar
import trackers.sources.trackleaders
- import trackers.sources.matrix
- import trackers.sources.garmin_inreach
modules = (
trackers.sources.traccar.config,
@@ -19,12 +19,12 @@ async def config_modules(app, settings):
)
source_trackers = {
- 'traccar': trackers.sources.traccar.start_event_tracker,
- 'spot': trackers.sources.spot.start_event_tracker,
- 'tkstorage': trackers.sources.tkstorage.start_event_tracker,
- 'trackleaders': trackers.sources.trackleaders.start_event_tracker,
- 'matrix': trackers.sources.matrix.start_event_tracker,
- 'garmin_inreach': trackers.sources.garmin_inreach.start_event_tracker,
+ "traccar": trackers.sources.traccar.start_event_tracker,
+ "spot": trackers.sources.spot.start_event_tracker,
+ "tkstorage": trackers.sources.tkstorage.start_event_tracker,
+ "trackleaders": trackers.sources.trackleaders.start_event_tracker,
+ "matrix": trackers.sources.matrix.start_event_tracker,
+ "garmin_inreach": trackers.sources.garmin_inreach.start_event_tracker,
}
exit_stack = AsyncExitStack()
@@ -32,5 +32,5 @@ async def config_modules(app, settings):
for module in modules:
await exit_stack.enter_async_context(module(app, settings))
- app['start_event_trackers'].update(source_trackers)
+ app["start_event_trackers"].update(source_trackers)
return exit_stack
diff --git a/trackers/persisted_func_cache.py b/trackers/persisted_func_cache.py
index 52f68c4..7ef804d 100644
--- a/trackers/persisted_func_cache.py
+++ b/trackers/persisted_func_cache.py
@@ -4,18 +4,17 @@
class PersistedFuncCache(object):
-
def __init__(self, path, func=None):
self.func = func
self.path = path
self.load()
self.unwritten_cache_items = []
- self.logger = logging.getLogger(f'persisted_func_cache.{path}')
+ self.logger = logging.getLogger(f"persisted_func_cache.{path}")
def load(self):
self.cache = {}
try:
- with open(self.path, 'rb') as f:
+ with open(self.path, "rb") as f:
unpacker = msgpack.Unpacker(f, use_list=False, raw=False)
while True:
@@ -25,7 +24,7 @@ def load(self):
except msgpack.exceptions.OutOfData:
break
except FileNotFoundError:
- open(self.path, 'wb').close()
+ open(self.path, "wb").close()
def __call__(self, *args, **kwargs):
key = self.key(*args, **kwargs)
@@ -46,10 +45,10 @@ def write_unwritten(self):
try:
items = self.unwritten_cache_items
self.unwritten_cache_items = []
- with open(self.path, 'ab') as f:
+ with open(self.path, "ab") as f:
msgpack.pack(items, f)
except Exception:
- self.logger.exception('Error writing unwritten: ')
+ self.logger.exception("Error writing unwritten: ")
def key(self, *args, **kwargs):
return args, tuple(sorted(kwargs.items()))
diff --git a/trackers/serve.py b/trackers/serve.py
index 680e316..043ec65 100644
--- a/trackers/serve.py
+++ b/trackers/serve.py
@@ -23,32 +23,36 @@
def main():
parser = trackers.bin_utils.get_base_argparser()
- parser.add_argument('--inet', action='store',
- help='Host address and port to listen on. (format: host:port)')
- parser.add_argument('--unix', action='store',
- help='Route of unix socket to listen on. ')
- parser.add_argument('--dev', action='store_true',
- help='Enable development tools (e.g. debug toolbar.)')
+ parser.add_argument(
+ "--inet",
+ action="store",
+ help="Host address and port to listen on. (format: host:port)",
+ )
+ parser.add_argument("--unix", action="store", help="Route of unix socket to listen on. ")
+ parser.add_argument(
+ "--dev",
+ action="store_true",
+ help="Enable development tools (e.g. debug toolbar.)",
+ )
args = parser.parse_args()
settings = trackers.bin_utils.get_combined_settings(defaults_yaml, args)
try:
-
if args.inet:
- host, _, port_str = args.inet.split(':')
+ host, _, port_str = args.inet.split(":")
port = int(port_str)
- settings['server_type'] = 'inet'
- settings['inet_host'] = host
- settings['inet_port'] = port
+ settings["server_type"] = "inet"
+ settings["inet_host"] = host
+ settings["inet_port"] = port
if args.unix:
- settings['server_type'] = 'unix'
- settings['unix_path'] = args.unix
+ settings["server_type"] = "unix"
+ settings["unix_path"] = args.unix
if args.dev:
- settings['debugtoolbar'] = True
- settings['aioserver_debug'] = True
+ settings["debugtoolbar"] = True
+ settings["aioserver_debug"] = True
if args.google_api_key:
- settings['google_api_key'] = args.google_api_key
+ settings["google_api_key"] = args.google_api_key
asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())
loop = asyncio.get_event_loop()
@@ -57,21 +61,23 @@ def main():
finally:
loop.close()
except Exception:
- logging.exception('Unhandled exception:')
+ logging.exception("Unhandled exception:")
sys.exit(3)
async def serve(loop, settings):
-
app = await trackers.web_app.make_aio_app(settings)
- runner = AppRunner(app, debug=settings.get('aioserver_debug', False),
- access_log_format='%l %u %t "%r" %s %b "%{Referrer}i" "%{User-Agent}i"')
+ runner = AppRunner(
+ app,
+ debug=settings.get("aioserver_debug", False),
+ access_log_format='%l %u %t "%r" %s %b "%{Referrer}i" "%{User-Agent}i"',
+ )
await runner.setup()
- if settings['server_type'] == 'inet':
- site = TCPSiteSocketName(runner, settings['inet_host'], settings['inet_port'])
- elif settings['server_type'] == 'unix':
- unix_path = settings['unix_path']
+ if settings["server_type"] == "inet":
+ site = TCPSiteSocketName(runner, settings["inet_host"], settings["inet_port"])
+ elif settings["server_type"] == "unix":
+ unix_path = settings["unix_path"]
if os.path.exists(unix_path):
try:
os.unlink(unix_path)
@@ -81,23 +87,23 @@ async def serve(loop, settings):
await site.start()
- if settings['server_type'] == 'unix':
- if 'unix_chmod' in settings:
- os.chmod(unix_path, settings['unix_chmod'])
- if 'unix_chown' in settings:
- shutil.chown(unix_path, **settings['unix_chown'])
+ if settings["server_type"] == "unix":
+ if "unix_chmod" in settings:
+ os.chmod(unix_path, settings["unix_chmod"])
+ if "unix_chown" in settings:
+ shutil.chown(unix_path, **settings["unix_chown"])
- logging.info(f'Serving on {site.name}')
+ logging.info(f"Serving on {site.name}")
try:
# Run forever (or we get interupt)
run_fut = asyncio.Future()
- for signame in ('SIGINT', 'SIGTERM'):
+ for signame in ("SIGINT", "SIGTERM"):
loop.add_signal_handler(getattr(signal, signame), run_fut.set_result, None)
try:
await run_fut
finally:
- for signame in ('SIGINT', 'SIGTERM'):
+ for signame in ("SIGINT", "SIGTERM"):
loop.remove_signal_handler(getattr(signal, signame))
finally:
await site.stop()
@@ -105,9 +111,8 @@ async def serve(loop, settings):
class TCPSiteSocketName(TCPSite):
-
@property
def name(self):
- scheme = 'https' if self._ssl_context else 'http'
+ scheme = "https" if self._ssl_context else "http"
socks = [sock.getsockname() for sock in self._server.sockets]
return [str(URL.build(scheme=scheme, host=sock[0], port=sock[1])) for sock in socks]
diff --git a/trackers/sources/garmin_inreach.py b/trackers/sources/garmin_inreach.py
index 1cf88ac..b890183 100644
--- a/trackers/sources/garmin_inreach.py
+++ b/trackers/sources/garmin_inreach.py
@@ -7,14 +7,14 @@
import aiohttp
from yarl import URL
-from trackers.base import print_tracker, Tracker
+from trackers.base import Tracker, print_tracker
logger = logging.getLogger(__name__)
@asynccontextmanager
async def config(app, settings):
- app['garmin_inreach.session'] = session = aiohttp.ClientSession(
+ app["garmin_inreach.session"] = session = aiohttp.ClientSession(
connector=aiohttp.TCPConnector(limit=1),
raise_for_status=True,
)
@@ -26,11 +26,18 @@ async def config(app, settings):
async def start_event_tracker(app, event, rider_name, tracker_data, start, end):
- return await start_tracker(app, rider_name, tracker_data['feed_id'], tracker_data.get('password'), start, end)
+ return await start_tracker(
+ app,
+ rider_name,
+ tracker_data["feed_id"],
+ tracker_data.get("password"),
+ start,
+ end,
+ )
async def start_tracker(app, tracker_name, feed_id, password, start, end):
- tracker = Tracker('garmin_inreach.{}-{}'.format(feed_id, tracker_name))
+ tracker = Tracker("garmin_inreach.{}-{}".format(feed_id, tracker_name))
monitor_task = asyncio.ensure_future(monitor_feed(app, tracker, feed_id, password, start, end))
tracker.stop = monitor_task.cancel
tracker.completed = monitor_task
@@ -44,8 +51,8 @@ async def monitor_feed(app, tracker, feed_id, password, start, end):
auth = aiohttp.BasicAuth(feed_id, password)
else:
auth = None
- session: aiohttp.ClientSession = app['garmin_inreach.session']
- url = URL(f'https://share.garmin.com/Feed/Share/{feed_id}')
+ session: aiohttp.ClientSession = app["garmin_inreach.session"]
+ url = URL(f"https://share.garmin.com/Feed/Share/{feed_id}")
if not start:
start = datetime.utcnow()
else:
@@ -58,13 +65,13 @@ async def monitor_feed(app, tracker, feed_id, password, start, end):
try:
now = datetime.utcnow()
if now > start:
- last = tracker.points[-1]['time'] if tracker.points else start
+ last = tracker.points[-1]["time"] if tracker.points else start
last = last.astimezone(timezone.utc)
- params = {'d1': last.isoformat(timespec='seconds') + 'z'}
+ params = {"d1": last.isoformat(timespec="seconds") + "z"}
if end and now > end:
- params['d2'] = end.isoformat(timespec='seconds') + 'z'
+ params["d2"] = end.isoformat(timespec="seconds") + "z"
url_with_params = url.update_query(params)
- tracker.logger.debug(f'Getting data. {url_with_params}')
+ tracker.logger.debug(f"Getting data. {url_with_params}")
async with session.get(url_with_params, auth=auth) as response:
kml_text = await response.text()
# tracker.logger.debug(f"Response: \n {kml_text}")
@@ -75,49 +82,53 @@ async def monitor_feed(app, tracker, feed_id, password, start, end):
except asyncio.CancelledError:
raise
except (aiohttp.client_exceptions.ClientError, RuntimeError) as e:
- tracker.logger.error('Error in monitor_feed: {!r}'.format(e))
+ tracker.logger.error("Error in monitor_feed: {!r}".format(e))
except Exception:
- tracker.logger.exception('Error in monitor_feed:')
+ tracker.logger.exception("Error in monitor_feed:")
await wait_for_next_check(tracker)
except asyncio.CancelledError:
raise
except Exception:
- tracker.logger.exception('Error in monitor_feed:')
+ tracker.logger.exception("Error in monitor_feed:")
async def process_data(tracker, kml_text, now, seen_ids):
xml_doc = xml.fromstring(kml_text)
kml_ns = {
- 'kml': 'http://www.opengis.net/kml/2.2',
+ "kml": "http://www.opengis.net/kml/2.2",
}
- placemarks = xml_doc.findall('./kml:Document/kml:Folder/kml:Placemark', kml_ns)
+ placemarks = xml_doc.findall("./kml:Document/kml:Folder/kml:Placemark", kml_ns)
new_points = []
for placemark in placemarks:
- extended_data = {data_el.attrib['name']: data_el.find('kml:value', kml_ns).text
- for data_el in placemark.findall('kml:ExtendedData/kml:Data', kml_ns)}
- if extended_data and extended_data['Id'] not in seen_ids:
- seen_ids.add(extended_data['Id'])
- lat = float(extended_data['Latitude'])
- lng = float(extended_data['Longitude'])
- elevation = float(extended_data['Elevation'].partition(' ')[0])
- time_utc = datetime.fromisoformat(placemark.find('kml:TimeStamp/kml:when', kml_ns).text[:-1]).replace(tzinfo=timezone.utc)
+ extended_data = {
+ data_el.attrib["name"]: data_el.find("kml:value", kml_ns).text
+ for data_el in placemark.findall("kml:ExtendedData/kml:Data", kml_ns)
+ }
+ if extended_data and extended_data["Id"] not in seen_ids:
+ seen_ids.add(extended_data["Id"])
+ lat = float(extended_data["Latitude"])
+ lng = float(extended_data["Longitude"])
+ elevation = float(extended_data["Elevation"].partition(" ")[0])
+ time_utc = datetime.fromisoformat(
+ placemark.find("kml:TimeStamp/kml:when", kml_ns).text[:-1]
+ ).replace(tzinfo=timezone.utc)
time = time_utc.astimezone().replace(tzinfo=None)
point = {
- 'position': [lat, lng, elevation],
- 'time': time,
- 'battery': None,
+ "position": [lat, lng, elevation],
+ "time": time,
+ "battery": None,
}
- if extended_data['Event'] == 'Tracking turned off from device.':
- point['tk_config'] = 'Off'
- if extended_data['Event'] == 'Tracking turned on from device.':
- point['tk_config'] = 'On'
+ if extended_data["Event"] == "Tracking turned off from device.":
+ point["tk_config"] = "Off"
+ if extended_data["Event"] == "Tracking turned on from device.":
+ point["tk_config"] = "On"
new_points.append(point)
- tracker.logger.debug(f'Got {len(new_points)} new points. {len(placemarks)} placemarks.')
+ tracker.logger.debug(f"Got {len(new_points)} new points. {len(placemarks)} placemarks.")
if new_points:
await tracker.new_points(new_points)
@@ -125,14 +136,14 @@ async def process_data(tracker, kml_text, now, seen_ids):
async def wait_for_next_check(tracker):
now = datetime.now()
if tracker.points:
- next_check_on_last_point_time = tracker.points[-1]['time'] + timedelta(minutes=11)
+ next_check_on_last_point_time = tracker.points[-1]["time"] + timedelta(minutes=11)
else:
next_check_on_last_point_time = datetime(year=1980, month=1, day=1)
next_check_on_now = now + timedelta(minutes=1)
next_check = max(next_check_on_now, next_check_on_last_point_time)
next_check_sec = (next_check - now).total_seconds()
- tracker.logger.debug(f'Next check: {next_check_sec} sec -- {next_check}')
+ tracker.logger.debug(f"Next check: {next_check_sec} sec -- {next_check}")
await asyncio.sleep(next_check_sec)
@@ -142,21 +153,21 @@ async def main():
app = {}
settings = {}
async with config(app, settings):
- tracker = await start_tracker(
- app, 'JanV', 'JanV', '', datetime(2019, 6, 17), None)
+ tracker = await start_tracker(app, "JanV", "JanV", "", datetime(2019, 6, 17), None)
print_tracker(tracker)
run_fut = asyncio.Future()
- for signame in ('SIGINT', 'SIGTERM'):
+ for signame in ("SIGINT", "SIGTERM"):
loop.add_signal_handler(getattr(signal, signame), run_fut.set_result, None)
try:
await run_fut
finally:
- for signame in ('SIGINT', 'SIGTERM'):
+ for signame in ("SIGINT", "SIGTERM"):
loop.remove_signal_handler(getattr(signal, signame))
tracker.stop()
await tracker.complete()
+
if __name__ == "__main__":
logging.basicConfig(level=logging.DEBUG)
loop = asyncio.get_event_loop()
diff --git a/trackers/sources/matrix.py b/trackers/sources/matrix.py
index b894983..0fa78cd 100644
--- a/trackers/sources/matrix.py
+++ b/trackers/sources/matrix.py
@@ -5,22 +5,32 @@
import aiohttp
from aniso8601 import parse_datetime
-from trackers.base import print_tracker, Tracker
+from trackers.base import Tracker, print_tracker
@asynccontextmanager
async def config(app, settings):
- async with aiohttp.ClientSession(connector=aiohttp.TCPConnector(limit=4)) as app['matrix.session']:
+ async with aiohttp.ClientSession(connector=aiohttp.TCPConnector(limit=4)) as app[
+ "matrix.session"
+ ]:
yield
def date_format(dt: datetime):
- return dt.strftime('%Y-%m-%dT%H_%M_%S')
+ return dt.strftime("%Y-%m-%dT%H_%M_%S")
async def start_event_tracker(app, event, rider_name, tracker_data, start, end):
- tracker = MatrixTracker(app, rider_name, tracker_data['username'], tracker_data['password'],
- tracker_data['client_id'], tracker_data['asset_id'], start, end)
+ tracker = MatrixTracker(
+ app,
+ rider_name,
+ tracker_data["username"],
+ tracker_data["password"],
+ tracker_data["client_id"],
+ tracker_data["asset_id"],
+ start,
+ end,
+ )
await tracker.start()
return tracker
@@ -33,10 +43,9 @@ async def start_event_tracker(app, event, rider_name, tracker_data, start, end):
class MatrixTracker(Tracker):
-
def __init__(self, app, tracker_name, username, password, client_id, asset_id, start, end):
- super().__init__('matrix.{}-{}'.format(asset_id, tracker_name))
- self.session: aiohttp.ClientSession = app['matrix.session']
+ super().__init__("matrix.{}-{}".format(asset_id, tracker_name))
+ self.session: aiohttp.ClientSession = app["matrix.session"]
self.username = username
self.password = password
self.client_id = client_id
@@ -56,30 +65,33 @@ async def start(self):
self.stop = self.monitor_task.cancel
async def login(self):
- self.logger.debug('Attempting Login.')
+ self.logger.debug("Attempting Login.")
login_data = {
- 'rememberMe': True,
- 'preferredLanguage': None,
- 'userName': self.username,
- 'password': self.password,
+ "rememberMe": True,
+ "preferredLanguage": None,
+ "userName": self.username,
+ "password": self.password,
}
- async with self.session.post('https://api-mit.mixtelematics.com/api/login', json=login_data) as response:
+ async with self.session.post(
+ "https://api-mit.mixtelematics.com/api/login", json=login_data
+ ) as response:
data = await response.json()
- if isinstance(data, str) and data.startswith('ErrorNo:'):
- self.logger.error('Login Failed')
+ if isinstance(data, str) and data.startswith("ErrorNo:"):
+ self.logger.error("Login Failed")
self.logged_in = False
# TODO Set status point, and maybe complete.
else:
self.logged_in = True
- self.headers = (('x-auth', data['authenticationToken']), )
+ self.headers = (("x-auth", data["authenticationToken"]),)
async def get(self, *args, **kwargs):
-
self.start_time, self.end_time or (datetime.now() + timedelta(days=1))
- response: aiohttp.ClientResponse = await self.session.get(*args, headers=self.headers, **kwargs)
- if response.status == 401 and response.reason == 'X-auth Not valid':
+ response: aiohttp.ClientResponse = await self.session.get(
+ *args, headers=self.headers, **kwargs
+ )
+ if response.status == 401 and response.reason == "X-auth Not valid":
response.release()
await self.login()
# retry
@@ -90,32 +102,42 @@ async def get(self, *args, **kwargs):
return await response.json()
async def get_points(self):
-
if not self.logged_in:
return
try:
start = self.last_get_points
end = self.end_time or (datetime.now() + timedelta(minutes=1))
- self.logger.debug(f'Getting points from {start} to {end} ({end - start}).')
+ self.logger.debug(f"Getting points from {start} to {end} ({end - start}).")
- url = f'https://api-mit.mixtelematics.com/api/tracking/trip/positions/client/{self.client_id}' \
- f'/asset/{self.asset_id}/fromDate/{date_format(start)}/toDate/{date_format(end)}'
+ url = (
+ f"https://api-mit.mixtelematics.com/api/tracking/trip/positions/client/{self.client_id}"
+ f"/asset/{self.asset_id}/fromDate/{date_format(start)}/toDate/{date_format(end)}"
+ )
- positions = (await self.get(url))['items']
+ positions = (await self.get(url))["items"]
new_points = []
new_position_ids = []
for position in positions:
- positionId = position['positionId']
+ positionId = position["positionId"]
if positionId not in self.seen_position_ids:
- time = parse_datetime(position['positionDateTime']['dateTime']).astimezone().replace(tzinfo=None)
+ time = (
+ parse_datetime(position["positionDateTime"]["dateTime"])
+ .astimezone()
+ .replace(tzinfo=None)
+ )
self.last_get_points = max(self.last_get_points, time)
- new_points.append({
- 'position': [position['position']['latitude'], position['position']['longitude']],
- 'time': time,
- 'server_time': datetime.now(),
- 'status': position['status'],
- })
+ new_points.append(
+ {
+ "position": [
+ position["position"]["latitude"],
+ position["position"]["longitude"],
+ ],
+ "time": time,
+ "server_time": datetime.now(),
+ "status": position["status"],
+ }
+ )
new_position_ids.append(positionId)
self.seen_position_ids.update(new_position_ids)
@@ -124,7 +146,7 @@ async def get_points(self):
except asyncio.CancelledError:
raise
except Exception:
- self.logger.exception('Error in get_points: ')
+ self.logger.exception("Error in get_points: ")
async def monitor(self):
try:
@@ -138,26 +160,34 @@ async def monitor(self):
async def main():
client_id = 93401
asset_id = 270834
- username = 'LEUENBERGER'
- password = ''
+ username = "LEUENBERGER"
+ password = ""
import signal
app = {}
settings = {}
async with config(app, settings):
-
- tracker = MatrixTracker(app, 'foobar', username, password, client_id, asset_id, datetime(2019, 3, 18), None)
+ tracker = MatrixTracker(
+ app,
+ "foobar",
+ username,
+ password,
+ client_id,
+ asset_id,
+ datetime(2019, 3, 18),
+ None,
+ )
await tracker.start()
print_tracker(tracker)
run_fut = asyncio.Future()
- for signame in ('SIGINT', 'SIGTERM'):
+ for signame in ("SIGINT", "SIGTERM"):
loop.add_signal_handler(getattr(signal, signame), run_fut.set_result, None)
try:
await run_fut
finally:
- for signame in ('SIGINT', 'SIGTERM'):
+ for signame in ("SIGINT", "SIGTERM"):
loop.remove_signal_handler(getattr(signal, signame))
tracker.stop()
await tracker.complete()
@@ -165,6 +195,7 @@ async def main():
if __name__ == "__main__":
import logging
+
logging.basicConfig(level=logging.DEBUG)
loop = asyncio.get_event_loop()
loop.run_until_complete(main())
diff --git a/trackers/sources/spot.py b/trackers/sources/spot.py
index 57e41b8..a7cea54 100644
--- a/trackers/sources/spot.py
+++ b/trackers/sources/spot.py
@@ -3,7 +3,6 @@
import logging
from contextlib import asynccontextmanager
from datetime import datetime, timedelta, timezone
-from itertools import chain
from pathlib import Path
from typing import cast
from xml.etree import ElementTree
@@ -11,27 +10,40 @@
import aiohttp
from aiohttp.web import Application as WebApplication
-from trackers.base import print_tracker, stream_store, Tracker, RateLimitingSemaphore
+from trackers.base import RateLimitingSemaphore, Tracker, print_tracker, stream_store
logger = logging.getLogger(__name__)
# https://www.findmespot.com/en-us/support/spot-trace/get-help/general/spot-api-support
+
@asynccontextmanager
async def config(app, settings):
- app['spot.session'] = session = aiohttp.ClientSession(connector=aiohttp.TCPConnector(limit=1), raise_for_status=True)
- app['spot.rate_limit_sem'] = RateLimitingSemaphore(0.5)
+ app["spot.session"] = session = aiohttp.ClientSession(
+ connector=aiohttp.TCPConnector(limit=1), raise_for_status=True
+ )
+ app["spot.rate_limit_sem"] = RateLimitingSemaphore(0.5)
if isinstance(app, WebApplication):
import trackers.web_app
- app.router.add_route('GET', '/spot/{feed_id}',
- handler=trackers.web_app.individual_page,
- name='spot_individual_page')
- app.router.add_route('GET', '/spot/{feed_id}/websocket',
- handler=functools.partial(trackers.web_app.individual_ws, get_individual_key,
- functools.partial(start_individual_tracker, app, settings)),
- name='spot_individual_ws')
+
+ app.router.add_route(
+ "GET",
+ "/spot/{feed_id}",
+ handler=trackers.web_app.individual_page,
+ name="spot_individual_page",
+ )
+ app.router.add_route(
+ "GET",
+ "/spot/{feed_id}/websocket",
+ handler=functools.partial(
+ trackers.web_app.individual_ws,
+ get_individual_key,
+ functools.partial(start_individual_tracker, app, settings),
+ ),
+ name="spot_individual_ws",
+ )
try:
yield
@@ -44,17 +56,17 @@ def get_individual_key(request):
async def start_individual_tracker(app, settings, request):
- feed_id = request.match_info['feed_id']
+ feed_id = request.match_info["feed_id"]
start = datetime.now() - timedelta(days=7)
- return await start_tracker(app, 'individual', feed_id, start, None)
+ return await start_tracker(app, "individual", feed_id, start, None)
async def start_event_tracker(app, event, rider_name, tracker_data, start, end):
- return await start_tracker(app, rider_name, tracker_data['feed_id'], start, end)
+ return await start_tracker(app, rider_name, tracker_data["feed_id"], start, end)
async def start_tracker(app, tracker_name, feed_id, start, end):
- tracker = Tracker('spot.{}-{}'.format(feed_id, tracker_name))
+ tracker = Tracker("spot.{}-{}".format(feed_id, tracker_name))
monitor_task = asyncio.ensure_future(monitor_feed(app, tracker, feed_id, start, end))
tracker.stop = monitor_task.cancel
tracker.completed = monitor_task
@@ -72,10 +84,17 @@ async def monitor_feed(app, tracker, feed_id, start: datetime, end: datetime):
last = start
# From this point on now, last, start, and end are all utc and tz naive.
- store_path = Path(app['trackers.settings']['cache_path'], 'spot', f'{feed_id}-{start.isoformat()}')
-
- with stream_store(store_path, tracker.logger) as (loaded_messages, write_messages):
- seen_ids = {message['id'] for message in loaded_messages}
+ store_path = Path(
+ app["trackers.settings"]["cache_path"],
+ "spot",
+ f"{feed_id}-{start.isoformat()}",
+ )
+
+ with stream_store(store_path, tracker.logger) as (
+ loaded_messages,
+ write_messages,
+ ):
+ seen_ids = {message["id"] for message in loaded_messages}
await messages_to_tracker(tracker, loaded_messages)
del loaded_messages
@@ -84,9 +103,15 @@ async def monitor_feed(app, tracker, feed_id, start: datetime, end: datetime):
now = datetime.utcnow().replace(tzinfo=None)
if now > start:
if tracker.points:
- last = (tracker.points[-1]['time'] + timedelta(seconds=1)).astimezone(timezone.utc).replace(tzinfo=None)
-
- new_messages = await get_new_messages(app, tracker, feed_id, start, end, last, seen_ids)
+ last = (
+ (tracker.points[-1]["time"] + timedelta(seconds=1))
+ .astimezone(timezone.utc)
+ .replace(tzinfo=None)
+ )
+
+ new_messages = await get_new_messages(
+ app, tracker, feed_id, start, end, last, seen_ids
+ )
if new_messages:
await messages_to_tracker(tracker, new_messages)
write_messages(new_messages)
@@ -96,25 +121,25 @@ async def monitor_feed(app, tracker, feed_id, start: datetime, end: datetime):
except asyncio.CancelledError:
raise
except (aiohttp.client_exceptions.ClientError, RuntimeError) as e:
- tracker.logger.error('Error in monitor_feed: {!r}'.format(e))
+ tracker.logger.error("Error in monitor_feed: {!r}".format(e))
except Exception:
- tracker.logger.exception('Error in monitor_feed:')
+ tracker.logger.exception("Error in monitor_feed:")
await wait_for_next_check(tracker)
except asyncio.CancelledError:
raise
except Exception:
- tracker.logger.exception('Error in monitor_feed:')
+ tracker.logger.exception("Error in monitor_feed:")
async def api_call(tracker, app, feed_id, params):
- url = f'https://api.findmespot.com/spot-main-web/consumer/rest-api/2.0/public/feed/{feed_id}/message.xml'
- session: aiohttp.ClientSession = app['spot.session']
- rate_limit_sem = app['spot.rate_limit_sem']
+ url = f"https://api.findmespot.com/spot-main-web/consumer/rest-api/2.0/public/feed/{feed_id}/message.xml"
+ session: aiohttp.ClientSession = app["spot.session"]
+ rate_limit_sem = app["spot.rate_limit_sem"]
async with rate_limit_sem:
async with session.get(url, params=params) as response:
- tracker.logger.debug(f'GET {response.url}')
+ tracker.logger.debug(f"GET {response.url}")
text = await response.text()
return ElementTree.fromstring(text)
@@ -123,31 +148,36 @@ def xml_to_dict(item: ElementTree):
return {child.tag: child.text for child in item}
-async def get_new_messages(app, tracker, feed_id, start: datetime, end: datetime, last: datetime, seen_ids):
+async def get_new_messages(
+ app, tracker, feed_id, start: datetime, end: datetime, last: datetime, seen_ids
+):
if last:
- base_params = {'startDate': last.isoformat(timespec='seconds') + '-0000'}
+ base_params = {"startDate": last.isoformat(timespec="seconds") + "-0000"}
else:
- base_params = {'startDate': start.isoformat(timespec='seconds') + '-0000'}
+ base_params = {"startDate": start.isoformat(timespec="seconds") + "-0000"}
if end:
- base_params['endDate'] = end.isoformat(timespec='seconds') + '-0000'
+ base_params["endDate"] = end.isoformat(timespec="seconds") + "-0000"
need_to_query_more = True
start = 0
query_results = []
while need_to_query_more:
- params = {**base_params, 'start': start}
+ params = {**base_params, "start": start}
tree: ElementTree = await api_call(tracker, app, feed_id, params)
- for error in tree.findall('errors/error'):
+ for error in tree.findall("errors/error"):
error = xml_to_dict(error)
- if error['code'] == 'E-0195':
+ if error["code"] == "E-0195":
level = logging.DEBUG
else:
level = logging.ERROR
- tracker.logger.log(msg=f"{error['text']} ({error['code']}): {error['description']}", level=level)
+ tracker.logger.log(
+ msg=f"{error['text']} ({error['code']}): {error['description']}",
+ level=level,
+ )
- count_el = tree.find('feedMessageResponse/count')
+ count_el = tree.find("feedMessageResponse/count")
if count_el is not None:
count = int(count_el.text)
else:
@@ -156,36 +186,42 @@ async def get_new_messages(app, tracker, feed_id, start: datetime, end: datetime
need_to_query_more = False
start += count
now = datetime.now(timezone.utc)
- messages = [xml_to_dict(item) for item in tree.findall('feedMessageResponse/messages/message')]
+ messages = [
+ xml_to_dict(item) for item in tree.findall("feedMessageResponse/messages/message")
+ ]
for message in messages:
- id = message['id']
+ id = message["id"]
if id not in seen_ids:
seen_ids.add(id)
- message['sever_time'] = now
- message['unixTime'] = int(message['unixTime'])
- message['latitude'] = float(message['latitude'])
- message['longitude'] = float(message['longitude'])
- message['altitude'] = float(message['altitude'])
+ message["sever_time"] = now
+ message["unixTime"] = int(message["unixTime"])
+ message["latitude"] = float(message["latitude"])
+ message["longitude"] = float(message["longitude"])
+ message["altitude"] = float(message["altitude"])
query_results.append(message)
- return list(sorted(query_results, key=lambda message: message['unixTime']))
+ return list(sorted(query_results, key=lambda message: message["unixTime"]))
async def messages_to_tracker(tracker, messages):
new_points = []
for message in messages:
- if message['altitude']:
- p = [message['latitude'], message['longitude'], message['altitude']]
+ if message["altitude"]:
+ p = [message["latitude"], message["longitude"], message["altitude"]]
else:
- p = [message['latitude'], message['longitude']]
- new_points.append({
- 'position': p,
- 'battery': message['batteryState'],
- # TODO We need to be able to specify the timezone in the config, as it seems to vary.
- 'time': datetime.fromtimestamp(message['unixTime']),
- 'server_time': cast(datetime, message['sever_time']).astimezone().replace(tzinfo=None),
- 'message_type': message['messageType'], # TODO translate into tracker status
- })
+ p = [message["latitude"], message["longitude"]]
+ new_points.append(
+ {
+ "position": p,
+ "battery": message["batteryState"],
+ # TODO We need to be able to specify the timezone in the config, as it seems to vary.
+ "time": datetime.fromtimestamp(message["unixTime"]),
+ "server_time": cast(datetime, message["sever_time"])
+ .astimezone()
+ .replace(tzinfo=None),
+ "message_type": message["messageType"], # TODO translate into tracker status
+ }
+ )
if new_points:
await tracker.new_points(new_points)
@@ -193,14 +229,16 @@ async def messages_to_tracker(tracker, messages):
async def wait_for_next_check(tracker):
now = datetime.now()
if tracker.points:
- next_check_on_last_point_time = tracker.points[-1]['time'] + timedelta(minutes=5, seconds=15)
+ next_check_on_last_point_time = tracker.points[-1]["time"] + timedelta(
+ minutes=5, seconds=15
+ )
else:
next_check_on_last_point_time = datetime(year=1980, month=1, day=1)
next_check_on_now = now + timedelta(minutes=2, seconds=30)
next_check = max(next_check_on_now, next_check_on_last_point_time)
next_check_sec = (next_check - now).total_seconds()
- tracker.logger.debug(f'Next check: {next_check_sec} sec -- {next_check}')
+ tracker.logger.debug(f"Next check: {next_check_sec} sec -- {next_check}")
await asyncio.sleep(next_check_sec)
@@ -208,27 +246,31 @@ async def main():
import signal
settings = {
- 'cache_path': 'cache',
- }
- app = {
- 'trackers.settings': settings
+ "cache_path": "cache",
}
+ app = {"trackers.settings": settings}
async with config(app, settings):
tracker = await start_tracker(
- app, 'foobar', '0Iejmjaumysr3dcqnMb96E3TnpaqlB60L', datetime(2023, 4, 8), None)
+ app,
+ "foobar",
+ "0Iejmjaumysr3dcqnMb96E3TnpaqlB60L",
+ datetime(2023, 4, 8),
+ None,
+ )
print_tracker(tracker)
run_fut = asyncio.Future()
- for signame in ('SIGINT', 'SIGTERM'):
+ for signame in ("SIGINT", "SIGTERM"):
loop.add_signal_handler(getattr(signal, signame), run_fut.set_result, None)
try:
await run_fut
finally:
- for signame in ('SIGINT', 'SIGTERM'):
+ for signame in ("SIGINT", "SIGTERM"):
loop.remove_signal_handler(getattr(signal, signame))
tracker.stop()
await tracker.complete()
+
if __name__ == "__main__":
logging.basicConfig(level=logging.DEBUG)
loop = asyncio.get_event_loop()
diff --git a/trackers/sources/tkstorage.py b/trackers/sources/tkstorage.py
index 9dfc210..6e0834e 100644
--- a/trackers/sources/tkstorage.py
+++ b/trackers/sources/tkstorage.py
@@ -14,10 +14,17 @@
import aiomsgpack
import more_itertools
-from aiohttp import web, WSMsgType
-
-from trackers.analyse import distance, Point
-from trackers.base import cancel_and_wait_task, list_register, Observable, print_tracker, run_forget_task, Tracker
+from aiohttp import WSMsgType, web
+
+from trackers.analyse import Point, distance
+from trackers.base import (
+ Observable,
+ Tracker,
+ cancel_and_wait_task,
+ list_register,
+ print_tracker,
+ run_forget_task,
+)
logger = logging.getLogger(__name__)
@@ -26,50 +33,80 @@
@asynccontextmanager
async def config(app, settings):
- app['tkstorage.points_received_observables'] = points_received_observables = defaultdict(partial(Observable, 'tkstorage.points_received'))
- app['tkstorage.send_queue'] = send_queue = asyncio.Queue()
- app['tkstorage.all_points_len'] = 0
- app['tkstorage.initial_download'] = initial_download_done = asyncio.Event()
- app['tkstorage.trackers_objects'] = trackers_objects = dict()
- app['tkstorage.values_changed'] = values_changed = Observable('tkstorage.values_changed')
- app['tkstorage.trackers'] = {}
- app['tkstorage.trackers_changed'] = trackers_changed = Observable('tkstorage.trackers_changed')
- app['tkstorage.sms_gateway_status'] = {}
- app['tkstorage.sms_gateway_status_changed'] = sms_gateway_status_changed = Observable('tkstorage.sms_gateway_status_changed')
- app['tkstorage.config'] = settings.get('tkstorage_config', True)
-
- connection_task = asyncio.ensure_future(connection(app, settings, points_received_observables, send_queue,
- initial_download_done, trackers_objects, values_changed))
+ app["tkstorage.points_received_observables"] = points_received_observables = defaultdict(
+ partial(Observable, "tkstorage.points_received")
+ )
+ app["tkstorage.send_queue"] = send_queue = asyncio.Queue()
+ app["tkstorage.all_points_len"] = 0
+ app["tkstorage.initial_download"] = initial_download_done = asyncio.Event()
+ app["tkstorage.trackers_objects"] = trackers_objects = dict()
+ app["tkstorage.values_changed"] = values_changed = Observable("tkstorage.values_changed")
+ app["tkstorage.trackers"] = {}
+ app["tkstorage.trackers_changed"] = trackers_changed = Observable("tkstorage.trackers_changed")
+ app["tkstorage.sms_gateway_status"] = {}
+ app["tkstorage.sms_gateway_status_changed"] = sms_gateway_status_changed = Observable(
+ "tkstorage.sms_gateway_status_changed"
+ )
+ app["tkstorage.config"] = settings.get("tkstorage_config", True)
+
+ connection_task = asyncio.ensure_future(
+ connection(
+ app,
+ settings,
+ points_received_observables,
+ send_queue,
+ initial_download_done,
+ trackers_objects,
+ values_changed,
+ )
+ )
if isinstance(app, web.Application):
import trackers.web_app
+
global web_app
web_app = trackers.web_app
- app['tkstorage.admin_ws_sessions'] = []
+ app["tkstorage.admin_ws_sessions"] = []
values_changed.subscribe(partial(send_values_changed_to_admin_ws, app))
trackers_changed.subscribe(partial(send_trackers_changed_to_admin_ws, app))
- sms_gateway_status_changed.subscribe(partial(send_sms_gateway_status_changed_to_admin_ws, app))
-
- app.router.add_route('GET', '/tkstorage_admin/tkstorage_websocket', handler=admin_ws, name='tkstorage_admin_ws')
- app.router.add_route('GET', '/tk/{id}',
- handler=trackers.web_app.individual_page,
- name='tkstorage_individual_page')
- app.router.add_route('GET', '/tk/{id}/websocket',
- handler=partial(trackers.web_app.individual_ws, get_individual_key,
- partial(start_individual_tracker, app, settings)),
- name='tkstorage_individual_ws')
+ sms_gateway_status_changed.subscribe(
+ partial(send_sms_gateway_status_changed_to_admin_ws, app)
+ )
+
+ app.router.add_route(
+ "GET",
+ "/tkstorage_admin/tkstorage_websocket",
+ handler=admin_ws,
+ name="tkstorage_admin_ws",
+ )
+ app.router.add_route(
+ "GET",
+ "/tk/{id}",
+ handler=trackers.web_app.individual_page,
+ name="tkstorage_individual_page",
+ )
+ app.router.add_route(
+ "GET",
+ "/tk/{id}/websocket",
+ handler=partial(
+ trackers.web_app.individual_ws,
+ get_individual_key,
+ partial(start_individual_tracker, app, settings),
+ ),
+ name="tkstorage_individual_ws",
+ )
try:
yield
finally:
- logger.debug('Shutdown.')
+ logger.debug("Shutdown.")
await cancel_and_wait_task(connection_task)
for tracker_objects in trackers_objects.values():
await tracker_objects.stop_config_apply_loop()
def get_tracker_objects(app, tk_id):
- trackers_objects = app['tkstorage.trackers_objects']
+ trackers_objects = app["tkstorage.trackers_objects"]
tracker_objects = trackers_objects.get(tk_id)
if not tracker_objects:
tracker_objects = TrackerObjects(app, tk_id)
@@ -93,9 +130,11 @@ class TrackerObjects(object):
desired_configs_changed: asyncio.Event = field(default_factory=asyncio.Event)
def get_highest_rank_desired_config(self):
- sorted_configs = sorted(self.desired_configs.keys(),
- key=lambda config_id: self.desired_configs[config_id].rank,
- reverse=True)
+ sorted_configs = sorted(
+ self.desired_configs.keys(),
+ key=lambda config_id: self.desired_configs[config_id].rank,
+ reverse=True,
+ )
return more_itertools.first(sorted_configs, None)
# TODO only allow one tracker to apply config.
@@ -113,26 +152,30 @@ def _desired_changed(self):
if self.desired_configs:
highest_rank_config_id = self.get_highest_rank_desired_config()
desired_config = self.desired_configs[highest_rank_config_id].config
- self.values['desired_config_text'] = config_text_from_config(desired_config)
+ self.values["desired_config_text"] = config_text_from_config(desired_config)
else:
- self.values['desired_config_text'] = ''
+ self.values["desired_config_text"] = ""
- self.values['desired_configs'] = self.desired_configs
- run_forget_task(self.app['tkstorage.values_changed']({self.tk_id: self.values}))
+ self.values["desired_configs"] = self.desired_configs
+ run_forget_task(self.app["tkstorage.values_changed"]({self.tk_id: self.values}))
def add_active_tracker(self, name):
- self.active_trackers.update((name, ))
+ self.active_trackers.update((name,))
self._update_active_tracker()
def remove_active_tracker(self, name):
- self.active_trackers.subtract((name, ))
+ self.active_trackers.subtract((name,))
self._update_active_tracker()
def _update_active_tracker(self):
- self.values['active'] = [tracker for tracker, count in self.active_trackers.items() if count]
- self.values['prev_active'] = [tracker for tracker, count in self.active_trackers.items() if not count]
+ self.values["active"] = [
+ tracker for tracker, count in self.active_trackers.items() if count
+ ]
+ self.values["prev_active"] = [
+ tracker for tracker, count in self.active_trackers.items() if not count
+ ]
- run_forget_task(self.app['tkstorage.values_changed']({self.tk_id: self.values}))
+ run_forget_task(self.app["tkstorage.values_changed"]({self.tk_id: self.values}))
def start_config_apply_loop(self):
self.config_apply_loop_fut = asyncio.ensure_future(self.config_apply_loop())
@@ -155,13 +198,13 @@ async def config_apply_loop(self):
delay = min(delay * 4, 1800)
apply_count += 1
- commands = await self.apply_config('first' if apply_count % 4 == 0 else False)
+ commands = await self.apply_config("first" if apply_count % 4 == 0 else False)
self._desired_changed()
- waits = (self.desired_configs_changed.wait(), )
+ waits = (self.desired_configs_changed.wait(),)
if commands:
- waits = waits + (asyncio.sleep(delay), )
+ waits = waits + (asyncio.sleep(delay),)
waits = tuple(asyncio.ensure_future(fut) for fut in waits)
try:
await asyncio.wait(waits, return_when=asyncio.FIRST_COMPLETED)
@@ -172,7 +215,7 @@ async def config_apply_loop(self):
except asyncio.CancelledError:
raise
except Exception:
- logger.exception('')
+ logger.exception("")
async def apply_config(self, urgent):
if self.desired_configs:
@@ -180,38 +223,40 @@ async def apply_config(self, urgent):
desired_config = self.desired_configs[highest_rank_config_id].config
commands = []
- desired_routetrack = desired_config.get('routetrack', False)
- if desired_routetrack != self.config.get('routetrack', False):
+ desired_routetrack = desired_config.get("routetrack", False)
+ if desired_routetrack != self.config.get("routetrack", False):
if desired_routetrack == True: # NOQA
desired_routetrack = 99
if desired_routetrack:
- commands.append(f'*routetrack*{desired_routetrack}*')
+ commands.append(f"*routetrack*{desired_routetrack}*")
else:
- commands.append('*routetrackoff*')
+ commands.append("*routetrackoff*")
if desired_routetrack:
- desired_rupload = desired_config['rupload']
- if desired_rupload != self.config.get('rupload'):
- commands.append(f'*rupload*{desired_rupload}*')
- desired_rsampling = desired_config['rsampling']
- if desired_rsampling != self.config.get('rsampling'):
- commands.append(f'*rsampling*{desired_rsampling}*')
-
- desired_check = desired_config.get('check', False)
- if desired_check != self.config.get('check', False):
+ desired_rupload = desired_config["rupload"]
+ if desired_rupload != self.config.get("rupload"):
+ commands.append(f"*rupload*{desired_rupload}*")
+ desired_rsampling = desired_config["rsampling"]
+ if desired_rsampling != self.config.get("rsampling"):
+ commands.append(f"*rsampling*{desired_rsampling}*")
+
+ desired_check = desired_config.get("check", False)
+ if desired_check != self.config.get("check", False):
if desired_check:
- commands.append(f'*checkm*{desired_check}*')
+ commands.append(f"*checkm*{desired_check}*")
else:
- commands.append('*checkoff*')
+ commands.append("*checkoff*")
if commands:
- verb = 'Sending' if self.app['tkstorage.config'] else 'Would send'
- action = f'{verb} commands {commands} Urgent: {urgent}'
+ verb = "Sending" if self.app["tkstorage.config"] else "Would send"
+ action = f"{verb} commands {commands} Urgent: {urgent}"
else:
- action = 'No action'
- logger.debug(f'{self.tk_id}: Desired: {desired_config} Current: {self.config} {action}')
- if commands and self.app['tkstorage.config']:
- await self.app['tkstorage.send_queue'].put({'id': self.tk_id, 'commands': commands, 'urgent': urgent})
+ action = "No action"
+ logger.debug(f"{self.tk_id}: Desired: {desired_config} Current: {self.config} {action}")
+ if commands and self.app["tkstorage.config"]:
+ await self.app["tkstorage.send_queue"].put(
+ {"id": self.tk_id, "commands": commands, "urgent": urgent}
+ )
return commands
@@ -222,46 +267,60 @@ class DesiredConfig(object):
rank: int = field(default=0)
-tk_id_key = lambda point: point['tk_id']
-time_key = lambda point: point.get('time') or point.get('server_time')
+tk_id_key = lambda point: point["tk_id"]
+time_key = lambda point: point.get("time") or point.get("server_time")
-async def connection(app, settings, points_received_observables, send_queue,
- initial_download_done, trackers_objects, values_changed):
+async def connection(
+ app,
+ settings,
+ points_received_observables,
+ send_queue,
+ initial_download_done,
+ trackers_objects,
+ values_changed,
+):
try:
reconnect_sleep_time = 5
connect_error_shown = False
- path = settings['tkstorage_path']
+ path = settings["tkstorage_path"]
loop = asyncio.get_event_loop()
while True:
try:
- logger.debug(f'Connecting to {path}')
+ logger.debug(f"Connecting to {path}")
try:
_, proto = await loop.create_unix_connection(
- aiomsgpack.make_msgpack_protocol_factory(loop=loop, unpacker_args={'raw': False},),
+ aiomsgpack.make_msgpack_protocol_factory(
+ loop=loop,
+ unpacker_args={"raw": False},
+ ),
path=path,
)
except FileNotFoundError as e:
- logger.error(f'{e}: {path}')
+ logger.error(f"{e}: {path}")
break
try:
connect_error_shown = False
reconnect_sleep_time = 1
- proto.write({'start': app['tkstorage.all_points_len']})
+ proto.write({"start": app["tkstorage.all_points_len"]})
write_fut = asyncio.ensure_future(write(app, proto, send_queue))
try:
async for msg in proto:
if isinstance(msg, dict):
- if 'trackers' in msg:
- app['tkstorage.trackers'] = msg['trackers']
- await app['tkstorage.trackers_changed'](msg['trackers'])
- for tracker in app['tkstorage.trackers']:
- app['tkstorage.points_received_observables'][tracker] = Observable(f'tkstorage.points_received.{tracker}')
- if 'sms_gateway_status' in msg:
- app['tkstorage.sms_gateway_status'] = msg['sms_gateway_status']
- await app['tkstorage.sms_gateway_status_changed'](msg['sms_gateway_status'])
+ if "trackers" in msg:
+ app["tkstorage.trackers"] = msg["trackers"]
+ await app["tkstorage.trackers_changed"](msg["trackers"])
+ for tracker in app["tkstorage.trackers"]:
+ app["tkstorage.points_received_observables"][
+ tracker
+ ] = Observable(f"tkstorage.points_received.{tracker}")
+ if "sms_gateway_status" in msg:
+ app["tkstorage.sms_gateway_status"] = msg["sms_gateway_status"]
+ await app["tkstorage.sms_gateway_status_changed"](
+ msg["sms_gateway_status"]
+ )
if isinstance(msg, list):
- logger.debug(f'Downloaded {len(msg)} points.')
+ logger.debug(f"Downloaded {len(msg)} points.")
new_points = []
for item in msg:
try:
@@ -269,31 +328,50 @@ async def connection(app, settings, points_received_observables, send_queue,
except Exception as e:
msg = copy(item)
msg[1] = datetime.fromtimestamp(msg[1])
- logger.error(f'Error in msg_item_to_point: msg={msg!r} \n {e}')
+ logger.error(
+ f"Error in msg_item_to_point: msg={msg!r} \n {e}"
+ )
point = None
# 2019-01-26 16:20:25,339 ERROR [trackers.sources.tkstorage] Error in msg_item_to_point: msg=[2916, 1548434791.840158, 1, '(864768011468128,DW3B,000000,A,2751.53042S,02740.57318E,5.931,164603,190.00,1470.40,11,0)', 'TK13'] month must be in 1..12
# 2019-01-26 16:20:26,874 ERROR [trackers.sources.tkstorage] Error in msg_item_to_point: msg=[11115, 1548481354.80168, 1, '(864768011468284,DW3B,000000,A,2738.40946S,02758.44679E,3.529,054234,113.02,1602.00,13,0)', 'TK28'] month must be in 1..12
# 2019-01-26 16:20:27,112 ERROR [trackers.sources.tkstorage] Error in msg_item_to_point: msg=[12537, 1548493482.802952, 1, '(864768011468169,DW30,000000,A,2751.23359S,02800.88446E,16.810,090045,228.83,1628.60,12,0)', 'TK10'] month must be in 1..12
- if point and point.get('tk_id'):
+ if point and point.get("tk_id"):
new_points.append(point)
new_points.sort(key=time_key)
- new_points = [consume_config_from_point(app, point) for point in new_points]
+ new_points = [
+ consume_config_from_point(app, point) for point in new_points
+ ]
trackers_values_changed = set()
for point in new_points:
- tk_id = point['tk_id']
+ tk_id = point["tk_id"]
trackers_values_changed.add(tk_id)
- tracker_values = get_tracker_objects(app, point['tk_id']).values
- tracker_values['last_connection'] = point['server_time']
- for key in ('tk_status', 'tk_config', 'position', 'battery'):
+ tracker_values = get_tracker_objects(app, point["tk_id"]).values
+ tracker_values["last_connection"] = point["server_time"]
+ for key in (
+ "tk_status",
+ "tk_config",
+ "position",
+ "battery",
+ ):
if key in point:
- tracker_values[key] = {'value': point[key], 'time': point['time']}
- await values_changed({tk_id: trackers_objects[tk_id].values for tk_id in trackers_values_changed})
-
- for tk_id, tracker_points in groupby(sorted(new_points, key=tk_id_key), key=tk_id_key):
+ tracker_values[key] = {
+ "value": point[key],
+ "time": point["time"],
+ }
+ await values_changed(
+ {
+ tk_id: trackers_objects[tk_id].values
+ for tk_id in trackers_values_changed
+ }
+ )
+
+ for tk_id, tracker_points in groupby(
+ sorted(new_points, key=tk_id_key), key=tk_id_key
+ ):
observable = points_received_observables.get(tk_id)
tracker_points = list(tracker_points)
trackers_objects[tk_id].points.extend(tracker_points)
@@ -311,19 +389,19 @@ async def connection(app, settings, points_received_observables, send_queue,
raise
except ConnectionRefusedError as e:
if not connect_error_shown:
- logger.error(f'Error in connection task: {e}')
+ logger.error(f"Error in connection task: {e}")
connect_error_shown = True
else:
- logger.debug(f'Error in connection task: {e}')
+ logger.debug(f"Error in connection task: {e}")
except Exception:
- logger.exception('Error in connection task: ')
- logger.debug('Reconnecting in {} sec'.format(reconnect_sleep_time))
+ logger.exception("Error in connection task: ")
+ logger.debug("Reconnecting in {} sec".format(reconnect_sleep_time))
await asyncio.sleep(reconnect_sleep_time)
reconnect_sleep_time = min((reconnect_sleep_time * 2, 30))
except asyncio.CancelledError:
raise
except Exception:
- logger.exception('Error in connection task: ')
+ logger.exception("Error in connection task: ")
async def write(app, proto, send_queue):
@@ -331,6 +409,7 @@ async def write(app, proto, send_queue):
msg = await send_queue.get()
proto.write(msg)
+
CONNECT = 0
MESSAGE_RECEIVED = 1
MESSAGE_SENT = 1
@@ -338,12 +417,12 @@ async def write(app, proto, send_queue):
CANCELED = 3
zc20_battery_levels = {
- '6': 100,
- '5': 80,
- '4': 50,
- '3': 20,
- '2': 10,
- '1': 0,
+ "6": 100,
+ "5": 80,
+ "4": 50,
+ "3": 20,
+ "2": 10,
+ "1": 0,
}
@@ -360,15 +439,15 @@ def msg_item_to_point(msg_item):
if type == MESSAGE_RECEIVED and data:
server_time = datetime.fromtimestamp(server_time)
point = {
- 'server_time': server_time,
- 'tk_id': id,
+ "server_time": server_time,
+ "tk_id": id,
}
- assert data[0] == '('
- assert data[-1] == ')'
+ assert data[0] == "("
+ assert data[-1] == ")"
data = data_split(data[1:-1])
msg_code = data[1]
- if msg_code == 'ZC20':
- point['time'] = parse_date_time(data[2], data[3], server_time)
+ if msg_code == "ZC20":
+ point["time"] = parse_date_time(data[2], data[3], server_time)
# battery_level = zc20_battery_levels.get(data[4])
battery_voltage = int(data[5])
# power_voltage = int(data[6])
@@ -376,89 +455,91 @@ def msg_item_to_point(msg_item):
# if battery_level is not None:
# point['battery'] = battery_level
if battery_voltage != 65535:
- point['battery_voltage'] = battery_voltage * 0.01
- point['battery'] = ((battery_voltage * 0.01) - 3.4) / 0.75 * 100
+ point["battery_voltage"] = battery_voltage * 0.01
+ point["battery"] = ((battery_voltage * 0.01) - 3.4) / 0.75 * 100
- if msg_code[:3] == 'DW3' and data[3] == 'A':
- lat = parse_coordinate(data[4], 'S')
- lng = parse_coordinate(data[5], 'W')
+ if msg_code[:3] == "DW3" and data[3] == "A":
+ lat = parse_coordinate(data[4], "S")
+ lng = parse_coordinate(data[5], "W")
alt = float(data[9])
- point['time'] = parse_date_time(data[2], data[7], server_time)
- point['position'] = (lat, lng, alt) if alt else (lat, lng)
+ point["time"] = parse_date_time(data[2], data[7], server_time)
+ point["position"] = (lat, lng, alt) if alt else (lat, lng)
- point['num_sat'] = int(data[10])
- if msg_code == 'ZC03':
- point['time'] = parse_date_time(data[2], data[3], server_time)
+ point["num_sat"] = int(data[10])
+ if msg_code == "ZC03":
+ point["time"] = parse_date_time(data[2], data[3], server_time)
msg = data[4]
# print(msg)
point.update(ZC03_parse(msg))
# HACK Some trackers are out by this time sometimes.
- if 'time' in point and (server_time - point['time']).total_seconds() // 100 == 713664:
- point['time'] = point['time'] + timedelta(days=826)
+ if "time" in point and (server_time - point["time"]).total_seconds() // 100 == 713664:
+ point["time"] = point["time"] + timedelta(days=826)
# HACK Some trackers are out by one week.
- if 'time' in point and (server_time.date() - point['time'].date()) == timedelta(weeks=1):
- point['time'] = point['time'] + timedelta(weeks=1)
+ if "time" in point and (server_time.date() - point["time"].date()) == timedelta(weeks=1):
+ point["time"] = point["time"] + timedelta(weeks=1)
return point
-status_battery_re = re.compile(r'Power: (\d*)%')
-check_re = re.compile(r'Check interval is set to (\d*) minute\(s\).')
-routetrack_re = re.compile(r'Routetrack (data is uploading|is on), Period is set to (\d*)')
-routetrack_time_re = re.compile(r'Notice: System has entered routetrack function for (\d*) hour\(s\).')
-rsampling_re = re.compile(r'Notice: Track sampling interval is (\d*) second\(s\).')
-rupload_re = re.compile(r'Notice: Upload time interval is set to (\d*) second\(s\)')
-check_on_re = re.compile(r'Notice: Check interval is set to (\d*) minute\(s\).')
+status_battery_re = re.compile(r"Power: (\d*)%")
+check_re = re.compile(r"Check interval is set to (\d*) minute\(s\).")
+routetrack_re = re.compile(r"Routetrack (data is uploading|is on), Period is set to (\d*)")
+routetrack_time_re = re.compile(
+ r"Notice: System has entered routetrack function for (\d*) hour\(s\)."
+)
+rsampling_re = re.compile(r"Notice: Track sampling interval is (\d*) second\(s\).")
+rupload_re = re.compile(r"Notice: Upload time interval is set to (\d*) second\(s\)")
+check_on_re = re.compile(r"Notice: Check interval is set to (\d*) minute\(s\).")
def ZC03_parse(msg):
status_battery_match = status_battery_re.search(msg)
if status_battery_match:
- yield 'battery', int(status_battery_match.group(1))
- yield 'tk_status', msg
+ yield "battery", int(status_battery_match.group(1))
+ yield "tk_status", msg
check_match = check_re.search(msg)
if check_match:
- yield 'tk_check', int(check_match.group(1))
+ yield "tk_check", int(check_match.group(1))
else:
- yield 'tk_check', False
+ yield "tk_check", False
routetrack_match = routetrack_re.search(msg)
if routetrack_match:
routetrack = int(routetrack_match.group(2))
if routetrack == 99:
routetrack = True
- yield 'tk_routetrack', routetrack
+ yield "tk_routetrack", routetrack
else:
- yield 'tk_routetrack', False
+ yield "tk_routetrack", False
else:
- if msg == 'Notice: System has ended routetrack function.':
- yield 'tk_routetrack', False
+ if msg == "Notice: System has ended routetrack function.":
+ yield "tk_routetrack", False
return
- if msg == 'Notice: Routetrack function is set to always on':
- yield 'tk_routetrack', True
+ if msg == "Notice: Routetrack function is set to always on":
+ yield "tk_routetrack", True
return
- if msg == 'Notice: System has ended check function.':
- yield 'tk_check', False
+ if msg == "Notice: System has ended check function.":
+ yield "tk_check", False
return
routetrack_time_match = routetrack_time_re.match(msg)
if routetrack_time_match:
- yield 'tk_routetrack', int(routetrack_time_match.group(1))
+ yield "tk_routetrack", int(routetrack_time_match.group(1))
return
rsampling_match = rsampling_re.match(msg)
if rsampling_match:
- yield 'tk_rsampling', int(rsampling_match.group(1))
+ yield "tk_rsampling", int(rsampling_match.group(1))
return
rupload_match = rupload_re.match(msg)
if rupload_match:
- yield 'tk_rupload', int(rupload_match.group(1))
+ yield "tk_rupload", int(rupload_match.group(1))
return
check_on_match = check_on_re.match(msg)
if check_on_match:
- yield 'tk_check', int(check_on_match.group(1))
+ yield "tk_check", int(check_on_match.group(1))
return
@@ -467,12 +548,12 @@ def data_split(data):
in_quote = False
last_i = 0
for i, c in enumerate(data):
- if c == '$':
+ if c == "$":
in_quote = not in_quote
- if c == ',' and not in_quote:
- split.append(data[last_i:i].strip('$'))
+ if c == "," and not in_quote:
+ split.append(data[last_i:i].strip("$"))
last_i = i + 1
- split.append(data[last_i:].strip('$'))
+ split.append(data[last_i:].strip("$"))
return split
@@ -485,14 +566,26 @@ def parse_date_time(date_raw: str, time_raw: str, server_time: datetime):
minutes = int(time_raw[2:4])
sec = int(time_raw[4:6])
- if date_raw == '000000':
+ if date_raw == "000000":
utc_server_time = server_time.astimezone(timezone.utc)
- utc_time00 = datetime(utc_server_time.year, utc_server_time.month, utc_server_time.day,
- hour, minutes, sec, tzinfo=timezone.utc)
+ utc_time00 = datetime(
+ utc_server_time.year,
+ utc_server_time.month,
+ utc_server_time.day,
+ hour,
+ minutes,
+ sec,
+ tzinfo=timezone.utc,
+ )
utc_timeP1 = utc_time00 + timedelta(days=1)
utc_timeM1 = utc_time00 + timedelta(days=-1)
- utc_time = min(utc_time00, utc_timeP1, utc_timeM1, key=lambda t: abs((utc_server_time - t).total_seconds()))
+ utc_time = min(
+ utc_time00,
+ utc_timeP1,
+ utc_timeM1,
+ key=lambda t: abs((utc_server_time - t).total_seconds()),
+ )
else:
day = int(date_raw[0:2])
month = int(date_raw[2:4])
@@ -503,52 +596,63 @@ def parse_date_time(date_raw: str, time_raw: str, server_time: datetime):
def parse_coordinate(raw, neg):
- split_index = raw.index('.') - 2
+ split_index = raw.index(".") - 2
deg = int(raw[0:split_index])
min = float(raw[split_index:-1])
hem = -1 if raw[-1] == neg else 1
return (deg + min / 60) * hem
-config_keys = {'tk_routetrack', 'tk_rupload', 'tk_rsampling', 'tk_check', }
+config_keys = {
+ "tk_routetrack",
+ "tk_rupload",
+ "tk_rsampling",
+ "tk_check",
+}
def config_text_from_config(c):
config_texts = []
- routetrack = c.get('routetrack')
+ routetrack = c.get("routetrack")
if routetrack:
- t = 'on' if routetrack == True else f'on for {routetrack} hrs' # NOQA
- if 'rupload' in c and 'rsampling' in c:
+ t = "on" if routetrack == True else f"on for {routetrack} hrs" # NOQA
+ if "rupload" in c and "rsampling" in c:
if c["rupload"] == c["rsampling"]:
config_texts.append(f'Routetrack {t} {c["rupload"]} sec upload')
else:
- config_texts.append(f'Routetrack {t} {c["rupload"]} sec upload, {c["rsampling"]} sec sample')
+ config_texts.append(
+ f'Routetrack {t} {c["rupload"]} sec upload, {c["rsampling"]} sec sample'
+ )
else:
- config_texts.append(f'Routetrack {t}')
- if c.get('check'):
+ config_texts.append(f"Routetrack {t}")
+ if c.get("check"):
config_texts.append(f'Check on {c["check"]} min upload')
if not config_texts:
- config_texts.append('Off')
- return '\n'.join(config_texts)
+ config_texts.append("Off")
+ return "\n".join(config_texts)
# TODO make this a method of TrackerObjects
def consume_config_from_point(app, point):
config_items = [(key[3:], value) for key, value in point.items() if key in config_keys]
if config_items:
- c = get_tracker_objects(app, point['tk_id']).config
+ c = get_tracker_objects(app, point["tk_id"]).config
c.update(config_items)
# Do we want to rm the config keys
point = copy(point)
- point['tk_config'] = config_text_from_config(c)
+ point["tk_config"] = config_text_from_config(c)
# print(repr(point['tk_config']))
return point
async def start_event_tracker(app, event, rider_name, tracker_data, start, end):
return await TKStorageTracker.start(
- app, rider_name, tracker_data['id'], start, end,
- tracker_data.get('config') or event.config.get('tk_config') or {},
+ app,
+ rider_name,
+ tracker_data["id"],
+ start,
+ end,
+ tracker_data.get("config") or event.config.get("tk_config") or {},
)
@@ -561,7 +665,7 @@ def get_individual_key(request):
async def start_individual_tracker(app, settings, request):
- id = request.match_info['id']
+ id = request.match_info["id"]
start = datetime.now() - timedelta(days=2)
print(id, start)
tracker = await TKStorageTracker.start(app, id, id, start, None)
@@ -570,30 +674,29 @@ async def start_individual_tracker(app, settings, request):
class TKStorageTracker(Tracker):
-
@classmethod
async def start(cls, app, tracker_name, id, start, end, config={}):
- tracker = cls(f'tkstorage.{id}-{tracker_name}')
+ tracker = cls(f"tkstorage.{id}-{tracker_name}")
tracker.tracker_name = tracker_name
tracker.app = app
tracker.id = id
tracker.start = start
tracker.end = end
tracker.config_read_start = start - timedelta(days=14)
- tracker.send_queue = app['tkstorage.send_queue']
+ tracker.send_queue = app["tkstorage.send_queue"]
tracker.config = config or {}
- tracker.config_rules = tracker.config.get('rules', [])
+ tracker.config_rules = tracker.config.get("rules", [])
tracker.objects = tracker_objects = get_tracker_objects(app, id)
tracker.objects.add_active_tracker(tracker_name)
# await app['tkstorage.initial_download'].wait()
try:
- await asyncio.wait_for(app['tkstorage.initial_download'].wait(), timeout=5)
+ await asyncio.wait_for(app["tkstorage.initial_download"].wait(), timeout=5)
except asyncio.TimeoutError:
- tracker.logger.error('Timeout waiting for initial download.')
+ tracker.logger.error("Timeout waiting for initial download.")
- tracker.points_received_observables = app['tkstorage.points_received_observables'][id]
+ tracker.points_received_observables = app["tkstorage.points_received_observables"][id]
tracker.points_received_observables.subscribe(tracker.points_received)
tracker.completed.add_done_callback(tracker.on_completed)
await tracker.points_received(tracker_objects.points)
@@ -602,7 +705,7 @@ async def start(cls, app, tracker_name, id, start, end, config={}):
now = datetime.now()
tracker.initial_config_task = None
- base_start = config.get('base', {}).get('start', start)
+ base_start = config.get("base", {}).get("start", start)
if config and base_start and now < end:
if now > start:
base_start = base_start + timedelta(seconds=60)
@@ -610,20 +713,24 @@ async def start(cls, app, tracker_name, id, start, end, config={}):
tracker.initial_config_task = run_forget_task(tracker.set_base_config(delay))
if end:
- tracker.complete_on_end_time_reached_task = run_forget_task(tracker.complete_on_end_time_reached())
+ tracker.complete_on_end_time_reached_task = run_forget_task(
+ tracker.complete_on_end_time_reached()
+ )
else:
tracker.complete_on_end_time_reached_task = None
return tracker
def use_point(self, point):
- time = point.get('time')
- server_time = point.get('server_time')
- if point['tk_id'] == 'TK24' and time == datetime(2019, 6, 23, 2, 0, 2):
+ time = point.get("time")
+ server_time = point.get("server_time")
+ if point["tk_id"] == "TK24" and time == datetime(2019, 6, 23, 2, 0, 2):
return False
if time_between(time, self.start, self.end):
return True
- if time_between(time, self.config_read_start, self.end) or time_between(server_time, self.config_read_start, self.end):
+ if time_between(time, self.config_read_start, self.end) or time_between(
+ server_time, self.config_read_start, self.end
+ ):
if any((key in point for key in config_keys)):
return True
# print(point)
@@ -634,20 +741,20 @@ async def points_received(self, points):
await self.new_points(points)
if self.config_rules:
- points_with_position = [point for point in points if 'position' in point]
+ points_with_position = [point for point in points if "position" in point]
if points_with_position:
- last_poistion = points_with_position[-1]['position']
+ last_poistion = points_with_position[-1]["position"]
for i, rule in enumerate(self.config_rules):
- if rule['type'] == 'within_dist_to_point':
- dist = distance(Point(*rule['point']), Point(*last_poistion[:2]))
- if dist < rule['dist']:
- self.logger.debug(f'Using config rule {i}')
- self.objects.add_desired_config('rules', rule['config'], rank=20)
+ if rule["type"] == "within_dist_to_point":
+ dist = distance(Point(*rule["point"]), Point(*last_poistion[:2]))
+ if dist < rule["dist"]:
+ self.logger.debug(f"Using config rule {i}")
+ self.objects.add_desired_config("rules", rule["config"], rank=20)
break
else:
- self.logger.debug('Clear config rule.')
- self.objects.del_desired_config('rules')
+ self.logger.debug("Clear config rule.")
+ self.objects.del_desired_config("rules")
def stop(self):
if not self.completed.done():
@@ -657,13 +764,13 @@ def set_finished(self):
super().set_finished()
if not self.completed.done():
for rule in self.config_rules:
- if rule['type'] == 'finished':
- self.objects.add_desired_config('finished', rule['config'], rank=30)
+ if rule["type"] == "finished":
+ self.objects.add_desired_config("finished", rule["config"], rank=30)
def reset_points(self):
super().reset_points()
- self.objects.del_desired_config('rules')
- self.objects.del_desired_config('finished')
+ self.objects.del_desired_config("rules")
+ self.objects.del_desired_config("finished")
def on_completed(self, fut):
if self.initial_config_task:
@@ -674,9 +781,9 @@ def on_completed(self, fut):
self.points_received_observables.unsubscribe(self.points_received)
- self.objects.del_desired_config('base_config')
- self.objects.del_desired_config('rules')
- self.objects.del_desired_config('finished')
+ self.objects.del_desired_config("base_config")
+ self.objects.del_desired_config("rules")
+ self.objects.del_desired_config("finished")
self.objects.remove_active_tracker(self.tracker_name)
@@ -685,49 +792,57 @@ def on_completed(self, fut):
async def complete_on_end_time_reached(self):
await asyncio.sleep((self.end - datetime.now()).total_seconds())
- await self.app['tkstorage.initial_download'].wait()
+ await self.app["tkstorage.initial_download"].wait()
self.completed.set_result(None)
async def set_base_config(self, delay):
await asyncio.sleep(delay)
- if self.config and 'base' in self.config:
- self.logger.debug('Setting base config')
- self.objects.add_desired_config('base_config', self.config['base'], rank=0)
+ if self.config and "base" in self.config:
+ self.logger.debug("Setting base config")
+ self.objects.add_desired_config("base_config", self.config["base"], rank=0)
async def admin_ws(request):
app = request.app
- send_queue = app['tkstorage.send_queue']
- trackers_objects = app['tkstorage.trackers_objects']
+ send_queue = app["tkstorage.send_queue"]
+ trackers_objects = app["tkstorage.trackers_objects"]
ws = web.WebSocketResponse()
ws.subscriptions = set()
await ws.prepare(request)
with contextlib.ExitStack() as exit_stack:
try:
- exit_stack.enter_context(list_register(app['trackers.ws_sessions'], ws))
- await web_app.message_to_multiple_wss(app, [ws], {
- 'values': {objects.tk_id: objects.values for objects in trackers_objects.values()},
- 'trackers': app['tkstorage.trackers'],
- 'sms_gateway_status': app['tkstorage.sms_gateway_status'],
- })
- exit_stack.enter_context(list_register(app['tkstorage.admin_ws_sessions'], ws))
+ exit_stack.enter_context(list_register(app["trackers.ws_sessions"], ws))
+ await web_app.message_to_multiple_wss(
+ app,
+ [ws],
+ {
+ "values": {
+ objects.tk_id: objects.values for objects in trackers_objects.values()
+ },
+ "trackers": app["tkstorage.trackers"],
+ "sms_gateway_status": app["tkstorage.sms_gateway_status"],
+ },
+ )
+ exit_stack.enter_context(list_register(app["tkstorage.admin_ws_sessions"], ws))
async for msg in ws:
if msg.type == WSMsgType.text:
try:
- logger.debug('receive: {}'.format(msg.data))
+ logger.debug("receive: {}".format(msg.data))
data = json.loads(msg.data)
- if 'commands' in data:
+ if "commands" in data:
await send_queue.put(data)
- if 'config' in data:
- tracker_objects = get_tracker_objects(app, data['id'])
- tracker_objects.add_desired_config('admin_console', data['config'], rank=100)
- if 'del_config' in data:
- tracker_objects = get_tracker_objects(app, data['id'])
- tracker_objects.del_desired_config('admin_console')
+ if "config" in data:
+ tracker_objects = get_tracker_objects(app, data["id"])
+ tracker_objects.add_desired_config(
+ "admin_console", data["config"], rank=100
+ )
+ if "del_config" in data:
+ tracker_objects = get_tracker_objects(app, data["id"])
+ tracker_objects.del_desired_config("admin_console")
except Exception:
- request.app['exception_recorder']()
- logger.exception('Error in receive ws msg:')
+ request.app["exception_recorder"]()
+ logger.exception("Error in receive ws msg:")
if msg.type == WSMsgType.close:
await ws.close()
@@ -736,23 +851,31 @@ async def admin_ws(request):
except asyncio.CancelledError:
pass
except Exception as e:
- request.app['exception_recorder']()
- await ws.close(message='Server Error: {}'.format(e))
- logger.exception('Error in tkstorage_admin_ws: ')
+ request.app["exception_recorder"]()
+ await ws.close(message="Server Error: {}".format(e))
+ logger.exception("Error in tkstorage_admin_ws: ")
finally:
return ws
async def send_values_changed_to_admin_ws(app, values_changed):
- await web_app.message_to_multiple_wss(app, app['tkstorage.admin_ws_sessions'], {'changed_values': values_changed})
+ await web_app.message_to_multiple_wss(
+ app, app["tkstorage.admin_ws_sessions"], {"changed_values": values_changed}
+ )
async def send_trackers_changed_to_admin_ws(app, trackers):
- await web_app.message_to_multiple_wss(app, app['tkstorage.admin_ws_sessions'], {'trackers': trackers})
+ await web_app.message_to_multiple_wss(
+ app, app["tkstorage.admin_ws_sessions"], {"trackers": trackers}
+ )
async def send_sms_gateway_status_changed_to_admin_ws(app, sms_gateway_status):
- await web_app.message_to_multiple_wss(app, app['tkstorage.admin_ws_sessions'], {'sms_gateway_status': sms_gateway_status})
+ await web_app.message_to_multiple_wss(
+ app,
+ app["tkstorage.admin_ws_sessions"],
+ {"sms_gateway_status": sms_gateway_status},
+ )
async def main():
@@ -760,11 +883,10 @@ async def main():
app = {}
settings = {
- 'tkstorage_path': os.path.expanduser('~/dev/trackers/tkstorage_watcher'),
+ "tkstorage_path": os.path.expanduser("~/dev/trackers/tkstorage_watcher"),
}
async with config(app, settings):
- tracker = await TKStorageTracker.start(
- app, 'gary', 'TK05', datetime(2021, 4, 10, 6), None)
+ tracker = await TKStorageTracker.start(app, "gary", "TK05", datetime(2021, 4, 10, 6), None)
print_tracker(tracker)
# await tracker.set_config({'check': 5})
# await tracker.set_config({'routetrack': False, 'check': False})
@@ -773,17 +895,19 @@ async def main():
# await asyncio.sleep(2)
import signal
+
run_fut = asyncio.Future()
- for signame in ('SIGINT', 'SIGTERM'):
+ for signame in ("SIGINT", "SIGTERM"):
loop.add_signal_handler(getattr(signal, signame), run_fut.set_result, None)
try:
await asyncio.wait((run_fut, tracker.completed), return_when=asyncio.FIRST_COMPLETED)
finally:
- for signame in ('SIGINT', 'SIGTERM'):
+ for signame in ("SIGINT", "SIGTERM"):
loop.remove_signal_handler(getattr(signal, signame))
tracker.stop()
await tracker.complete()
+
if __name__ == "__main__":
logging.basicConfig(level=logging.INFO)
loop = asyncio.get_event_loop()
diff --git a/trackers/sources/traccar.py b/trackers/sources/traccar.py
index 56df231..1015711 100644
--- a/trackers/sources/traccar.py
+++ b/trackers/sources/traccar.py
@@ -13,95 +13,121 @@
from aiohttp.web import Application as WebApplication
from aniso8601 import parse_datetime
-from trackers.base import Observable, print_tracker, Tracker
+from trackers.base import Observable, Tracker, print_tracker
logger = logging.getLogger(__name__)
@asynccontextmanager
async def config(app, settings):
- app['trackers.traccar_servers'] = servers = {}
- for server_name, server in settings['traccar_servers'].items():
- server['session'] = session = aiohttp.ClientSession(
+ app["trackers.traccar_servers"] = servers = {}
+ for server_name, server in settings["traccar_servers"].items():
+ server["session"] = session = aiohttp.ClientSession(
# auth=aiohttp.BasicAuth(*server['auth']),
connector=aiohttp.TCPConnector(limit=4),
raise_for_status=True,
)
- server['position_received_observables'] = position_received_observables = defaultdict(partial(Observable, logger))
- server['ws_task'] = asyncio.ensure_future(server_ws_task(app, settings, session, server_name, server, position_received_observables))
- server['login_lock'] = asyncio.Lock()
+ server["position_received_observables"] = position_received_observables = defaultdict(
+ partial(Observable, logger)
+ )
+ server["ws_task"] = asyncio.ensure_future(
+ server_ws_task(
+ app,
+ settings,
+ session,
+ server_name,
+ server,
+ position_received_observables,
+ )
+ )
+ server["login_lock"] = asyncio.Lock()
servers[server_name] = server
if isinstance(app, WebApplication):
import trackers.web_app
- app.router.add_route('GET', '/traccar/{unique_id}',
- handler=trackers.web_app.individual_page,
- name='tarccar_individual_page')
- app.router.add_route('GET', '/traccar/{unique_id}/websocket',
- handler=functools.partial(trackers.web_app.individual_ws, get_individual_key,
- functools.partial(start_individual_tracker, app, settings)),
- name='tarccar_individual_ws')
+
+ app.router.add_route(
+ "GET",
+ "/traccar/{unique_id}",
+ handler=trackers.web_app.individual_page,
+ name="tarccar_individual_page",
+ )
+ app.router.add_route(
+ "GET",
+ "/traccar/{unique_id}/websocket",
+ handler=functools.partial(
+ trackers.web_app.individual_ws,
+ get_individual_key,
+ functools.partial(start_individual_tracker, app, settings),
+ ),
+ name="tarccar_individual_ws",
+ )
try:
yield
finally:
- logger.debug('Shutdown.')
+ logger.debug("Shutdown.")
for server_name, server in servers.items():
- server['ws_task'].cancel()
+ server["ws_task"].cancel()
try:
- await server['ws_task']
+ await server["ws_task"]
except asyncio.CancelledError:
pass
except Exception:
- logger.exception('Error in ws_task: ')
+ logger.exception("Error in ws_task: ")
await logout(app, server_name)
- await server['session'].close()
+ await server["session"].close()
async def ensure_login(app, server_name):
- server = app['trackers.traccar_servers'][server_name]
- async with server['login_lock']:
- logger = logging.getLogger('{}.{}'.format(__name__, server_name))
- if not server.get('user_id'):
- session_response = await server['session'].post('{url}/api/session'.format_map(server), data={'email': [server['auth'][0]], 'password': [server['auth'][1]]})
+ server = app["trackers.traccar_servers"][server_name]
+ async with server["login_lock"]:
+ logger = logging.getLogger("{}.{}".format(__name__, server_name))
+ if not server.get("user_id"):
+ session_response = await server["session"].post(
+ "{url}/api/session".format_map(server),
+ data={"email": [server["auth"][0]], "password": [server["auth"][1]]},
+ )
user = await session_response.json()
- server['user_id'] = user['id']
- logger.info('Successfull login to {url}'.format_map(server))
+ server["user_id"] = user["id"]
+ logger.info("Successfull login to {url}".format_map(server))
async def logout(app, server_name):
- server = app['trackers.traccar_servers'][server_name]
- logger = logging.getLogger('{}.{}'.format(__name__, server_name))
- if server.get('user_id'):
+ server = app["trackers.traccar_servers"][server_name]
+ logger = logging.getLogger("{}.{}".format(__name__, server_name))
+ if server.get("user_id"):
try:
- await server['session'].delete('{}/api/session'.format(server['url']))
+ await server["session"].delete("{}/api/session".format(server["url"]))
except aiohttp.client_exceptions.ClientError as e:
- logger.error('Error in delete session: {!r}'.format(e))
+ logger.error("Error in delete session: {!r}".format(e))
except Exception:
- logger.exception('Error in delete session:')
- del server['user_id']
+ logger.exception("Error in delete session:")
+ del server["user_id"]
-async def server_ws_task(app, settings, session, server_name, server, position_received_observables):
+async def server_ws_task(
+ app, settings, session, server_name, server, position_received_observables
+):
try:
- url = '{}/api/socket'.format(server['url'])
- logger = logging.getLogger('{}.{}'.format(__name__, server_name))
+ url = "{}/api/socket".format(server["url"])
+ logger = logging.getLogger("{}.{}".format(__name__, server_name))
reconnect_sleep_time = 5
connect_error_shown = False
while True:
try:
await ensure_login(app, server_name)
- logger.debug('Connecting to ws {}'.format(url))
+ logger.debug("Connecting to ws {}".format(url))
async with session.ws_connect(url) as ws:
connect_error_shown = False
reconnect_sleep_time = 1
async for msg in ws:
if msg.type == aiohttp.WSMsgType.TEXT:
data = msg.json()
- if 'positions' in data:
- for position in data['positions']:
- device_id = position['deviceId']
+ if "positions" in data:
+ for position in data["positions"]:
+ device_id = position["deviceId"]
observable = position_received_observables.get(device_id)
if observable:
await observable(position)
@@ -113,25 +139,31 @@ async def server_ws_task(app, settings, session, server_name, server, position_r
raise
except aiohttp.client_exceptions.ClientError as e:
if not connect_error_shown:
- logger.error(f'Error in ws_task: {e}')
+ logger.error(f"Error in ws_task: {e}")
connect_error_shown = True
else:
- logger.debug(f'Error in ws_task: {e}')
+ logger.debug(f"Error in ws_task: {e}")
await logout(app, server_name)
except Exception:
- logger.exception('Error in ws_task: ')
- logger.debug('Reconnecting in {} sec'.format(reconnect_sleep_time))
+ logger.exception("Error in ws_task: ")
+ logger.debug("Reconnecting in {} sec".format(reconnect_sleep_time))
await asyncio.sleep(reconnect_sleep_time)
reconnect_sleep_time = min((reconnect_sleep_time * 2, 30))
except asyncio.CancelledError:
raise
except Exception:
- logger.exception('Error in ws_task: ')
+ logger.exception("Error in ws_task: ")
async def start_event_tracker(app, event, rider_name, tracker_data, start, end):
- return await start_tracker(app, rider_name, tracker_data.get('server', 'local'),
- tracker_data['unique_id'], start, end)
+ return await start_tracker(
+ app,
+ rider_name,
+ tracker_data.get("server", "local"),
+ tracker_data["unique_id"],
+ start,
+ end,
+ )
def get_individual_key(request):
@@ -139,63 +171,75 @@ def get_individual_key(request):
async def start_individual_tracker(app, settings, request):
- unique_id = request.match_info['unique_id']
- server_name = 'local'
+ unique_id = request.match_info["unique_id"]
+ server_name = "local"
start = datetime.now() - timedelta(days=1)
- return await start_tracker(app, unique_id,
- server_name, unique_id,
- start, None)
+ return await start_tracker(app, unique_id, server_name, unique_id, start, None)
async def start_tracker(app, tracker_name, server_name, device_unique_id, start, end):
device_unique_id = str(device_unique_id)
await ensure_login(app, server_name)
- server = app['trackers.traccar_servers'][server_name]
- server_url = server['url']
- session = server['session']
+ server = app["trackers.traccar_servers"][server_name]
+ server_url = server["url"]
+ session = server["session"]
- devices_response = await session.get(f'{server_url}/api/devices', params={'all': 'true'})
+ devices_response = await session.get(f"{server_url}/api/devices", params={"all": "true"})
devices = await devices_response.json()
try:
- device = more_itertools.first((device for device in devices if device['uniqueId'] == device_unique_id))
- device_id = device['id']
- if device['name'] == device['uniqueId']:
+ device = more_itertools.first(
+ (device for device in devices if device["uniqueId"] == device_unique_id)
+ )
+ device_id = device["id"]
+ if device["name"] == device["uniqueId"]:
# Update name on traccar
set_name_response = await session.put(
- f'{server_url}/api/devices/{device_id}',
- json={'name': tracker_name, 'uniqueId': device['uniqueId'], 'id': device['id']})
+ f"{server_url}/api/devices/{device_id}",
+ json={
+ "name": tracker_name,
+ "uniqueId": device["uniqueId"],
+ "id": device["id"],
+ },
+ )
await set_name_response.json()
except ValueError:
get_device_response = await session.post(
- f'{server_url}/api/devices',
- json={'uniqueId': device_unique_id, 'name': tracker_name})
- device_id = (await get_device_response.json())['id']
+ f"{server_url}/api/devices",
+ json={"uniqueId": device_unique_id, "name": tracker_name},
+ )
+ device_id = (await get_device_response.json())["id"]
- tracker = Tracker('traccar.{}.{}-{}'.format(server_name, device_unique_id, tracker_name))
+ tracker = Tracker("traccar.{}.{}-{}".format(server_name, device_unique_id, tracker_name))
try:
- await session.post(f'{server_url}/api/permissions',
- json={'userId': server['user_id'], 'deviceId': device_id})
+ await session.post(
+ f"{server_url}/api/permissions",
+ json={"userId": server["user_id"], "deviceId": device_id},
+ )
except Exception as e:
- tracker.logger.error(f'Error in set permissions: {e}')
+ tracker.logger.error(f"Error in set permissions: {e}")
tracker.server = server
tracker.device_id = device_id
- tracker.start = (start if start else (datetime.now() - timedelta(days=2)).replace(microsecond=0))
+ tracker.start = start if start else (datetime.now() - timedelta(days=2)).replace(microsecond=0)
tracker.end = end
tracker.seen_ids = seen_ids = set()
- initial_query_end = (end if end else (datetime.now() + timedelta(days=1)).replace(microsecond=0))
- positions_response = await session.get(f'{server_url}/api/positions', params={
- 'deviceId': device_id,
- 'from': tracker.start.astimezone(dateutil.tz.UTC).isoformat(),
- 'to': initial_query_end.astimezone(dateutil.tz.UTC).isoformat()
- }, headers={'Accept': 'application/json'})
+ initial_query_end = end if end else (datetime.now() + timedelta(days=1)).replace(microsecond=0)
+ positions_response = await session.get(
+ f"{server_url}/api/positions",
+ params={
+ "deviceId": device_id,
+ "from": tracker.start.astimezone(dateutil.tz.UTC).isoformat(),
+ "to": initial_query_end.astimezone(dateutil.tz.UTC).isoformat(),
+ },
+ headers={"Accept": "application/json"},
+ )
positions = await positions_response.json()
points = [traccar_position_translate(position) for position in positions]
- seen_ids.update([position['id'] for position in positions])
+ seen_ids.update([position["id"] for position in positions])
tracker.position_recived = functools.partial(tracker_position_received, tracker)
- server['position_received_observables'][device_id].subscribe(tracker.position_recived)
+ server["position_received_observables"][device_id].subscribe(tracker.position_recived)
await tracker.new_points(points)
tracker.finished = asyncio.Event()
@@ -204,15 +248,19 @@ async def start_tracker(app, tracker_name, server_name, device_unique_id, start,
tracker.completed.add_done_callback(functools.partial(tracker_on_completed, tracker))
if end:
asyncio.get_event_loop().call_at(
- asyncio.get_event_loop().time() - time.time() + end.timestamp(), tracker.finished.set)
+ asyncio.get_event_loop().time() - time.time() + end.timestamp(),
+ tracker.finished.set,
+ )
return tracker
async def tracker_position_received(tracker, position):
- if position['id'] not in tracker.seen_ids:
- tracker.seen_ids.add(position['id'])
+ if position["id"] not in tracker.seen_ids:
+ tracker.seen_ids.add(position["id"])
point = traccar_position_translate(position)
- if (not tracker.start or tracker.start < point['time']) and (not tracker.end or point['time'] < tracker.end):
+ if (not tracker.start or tracker.start < point["time"]) and (
+ not tracker.end or point["time"] < tracker.end
+ ):
await tracker.new_points([point])
@@ -221,64 +269,70 @@ def tracker_stop(tracker):
def tracker_on_completed(tracker, fut):
- tracker.server['position_received_observables'][tracker.device_id].unsubscribe(tracker.position_recived)
+ tracker.server["position_received_observables"][tracker.device_id].unsubscribe(
+ tracker.position_recived
+ )
async def tracker_finish(tracker):
await tracker.finished.wait()
- tracker.server['position_received_observables'][tracker.device_id].unsubscribe(tracker.position_recived)
+ tracker.server["position_received_observables"][tracker.device_id].unsubscribe(
+ tracker.position_recived
+ )
def traccar_position_translate(position):
- if position['altitude']:
- p = [position['latitude'], position['longitude'], position['altitude']]
+ if position["altitude"]:
+ p = [position["latitude"], position["longitude"], position["altitude"]]
else:
- p = [position['latitude'], position['longitude']]
+ p = [position["latitude"], position["longitude"]]
point = {
- 'position': p,
- 'accuracy': position['accuracy'],
- 'time': parse_datetime(position['fixTime']).astimezone().replace(tzinfo=None),
+ "position": p,
+ "accuracy": position["accuracy"],
+ "time": parse_datetime(position["fixTime"]).astimezone().replace(tzinfo=None),
# server_time is null in websocket positions :-( Need to log an issue, and fix it.
- 'server_time': (
- parse_datetime(position['serverTime']).astimezone().replace(tzinfo=None)
- if position['serverTime'] else datetime.now()),
+ "server_time": (
+ parse_datetime(position["serverTime"]).astimezone().replace(tzinfo=None)
+ if position["serverTime"]
+ else datetime.now()
+ ),
}
- if 'batteryLevel' in position['attributes']:
- point['battery'] = position['attributes'].get('batteryLevel')
- if 'battery' in position['attributes']:
- point['battery_voltage'] = position['attributes'].get('battery')
+ if "batteryLevel" in position["attributes"]:
+ point["battery"] = position["attributes"].get("batteryLevel")
+ if "battery" in position["attributes"]:
+ point["battery_voltage"] = position["attributes"].get("battery")
return point
async def main():
app = {}
settings = {
- 'traccar_servers': {
- 'trackrace_tk':
- {
- 'url': 'http://traccar.trackrace.co.za:8082',
- 'auth': ['admin', ''],
- }
+ "traccar_servers": {
+ "trackrace_tk": {
+ "url": "http://traccar.trackrace.co.za:8082",
+ "auth": ["admin", ""],
+ }
}
}
import signal
+
async with config(app, settings):
- tracker = await start_tracker(
- app, 'gary', 'trackrace_tk', 'garyvdm', None, None)
+ tracker = await start_tracker(app, "gary", "trackrace_tk", "garyvdm", None, None)
print_tracker(tracker)
# await tracker.finish()
run_fut = asyncio.Future()
- for signame in ('SIGINT', 'SIGTERM'):
+ for signame in ("SIGINT", "SIGTERM"):
loop.add_signal_handler(getattr(signal, signame), run_fut.set_result, None)
try:
await run_fut
finally:
- for signame in ('SIGINT', 'SIGTERM'):
+ for signame in ("SIGINT", "SIGTERM"):
loop.remove_signal_handler(getattr(signal, signame))
tracker.stop()
await tracker.complete()
+
if __name__ == "__main__":
logging.basicConfig(level=logging.DEBUG)
loop = asyncio.get_event_loop()
diff --git a/trackers/sources/trackleaders.py b/trackers/sources/trackleaders.py
index 60c0373..5dd211b 100644
--- a/trackers/sources/trackleaders.py
+++ b/trackers/sources/trackleaders.py
@@ -12,6 +12,7 @@
import dateutil.tz
from calmjs.parse import es5
from calmjs.parse.asttypes import Assign, ExprStatement, FunctionCall
+from dulwich_tree import TreeWriter
import trackers.events
from trackers.base import Tracker
@@ -24,7 +25,6 @@
process_secondary_route_details,
update_bounds_inner,
)
-from trackers.dulwich_helpers import TreeWriter
# TODO: proper start/end filtering.
@@ -33,7 +33,9 @@
@asynccontextmanager
async def config(app, settings):
- app['trackleaders.session'] = session = aiohttp.ClientSession(connector=aiohttp.TCPConnector(limit=4))
+ app["trackleaders.session"] = session = aiohttp.ClientSession(
+ connector=aiohttp.TCPConnector(limit=4)
+ )
# app['trackleaders.rate_limit_sem'] = asyncio.Semaphore()
try:
yield
@@ -43,9 +45,9 @@ async def config(app, settings):
def print_node(node):
print(repr(node)[:600])
- print([i for i in dir(node) if not i.startswith('_')])
+ print([i for i in dir(node) if not i.startswith("_")])
print(str(node)[:200])
- print('------------------------------------------------------------')
+ print("------------------------------------------------------------")
async def get(session, *args, **kwargs):
@@ -56,7 +58,7 @@ async def get(session, *args, **kwargs):
@async_command(event_command_parser, basic=True)
async def get_config(app, settings, args):
- tree_writer = TreeWriter(app['trackers.data_repo'])
+ tree_writer = TreeWriter(app["trackers.data_repo"])
event_name = event_name_clean(args.event_name, settings)
trackleaders_event_name = event_name
@@ -65,32 +67,42 @@ async def get_config(app, settings, args):
except KeyError:
event = await trackers.events.Event(app, event_name)
- logger.info('Downloading.')
+ logger.info("Downloading.")
async with aiohttp.ClientSession() as session:
- race_page_text = await get(session, f'http://trackleaders.com/{trackleaders_event_name}f.php')
- riders_list_text = await get(session, f'http://trackleaders.com/spot/{trackleaders_event_name}/sortlist.php')
- route_text = await get(session, f'http://trackleaders.com/spot/{trackleaders_event_name}/route.js')
- markers_text = await get(session, f'http://trackleaders.com/spot/{trackleaders_event_name}/checkgen.js')
-
- logger.info('Scraping.')
+ race_page_text = await get(
+ session, f"http://trackleaders.com/{trackleaders_event_name}f.php"
+ )
+ riders_list_text = await get(
+ session,
+ f"http://trackleaders.com/spot/{trackleaders_event_name}/sortlist.php",
+ )
+ route_text = await get(
+ session, f"http://trackleaders.com/spot/{trackleaders_event_name}/route.js"
+ )
+ markers_text = await get(
+ session,
+ f"http://trackleaders.com/spot/{trackleaders_event_name}/checkgen.js",
+ )
+
+ logger.info("Scraping.")
config = event.config
race_page = bs4.BeautifulSoup(race_page_text, "html.parser")
- config['title'] = race_page.find('title').string.partition(' live')[0]
+ config["title"] = race_page.find("title").string.partition(" live")[0]
# TODO Start time
riders_list = bs4.BeautifulSoup(riders_list_text, "html.parser")
- riders_links = riders_list.find_all('a', title=re.compile('Open .* full history'))
+ riders_links = riders_list.find_all("a", title=re.compile("Open .* full history"))
- config['riders'] = [
+ config["riders"] = [
{
- 'name': rider_link.string.strip(),
- 'name_short': rider_link.string.partition(' ')[0],
- 'tracker': {
- 'type': 'trackleaders',
- 'event': str(args.event_name),
- 'name': parse_qs(urlparse(rider_link['href']).query)['name'][0],
+ "name": rider_link.string.strip(),
+ "name_short": rider_link.string.partition(" ")[0],
+ "tracker": {
+ "type": "trackleaders",
+ "event": str(args.event_name),
+ "name": parse_qs(urlparse(rider_link["href"]).query)["name"][0],
},
}
for rider_link in riders_links
@@ -99,10 +111,11 @@ async def get_config(app, settings, args):
route_js = es5(route_text)
route_point_nodes = route_js.children()[0].elements[0].expr.right
route_points = [
- [float(str(subnode)) for subnode in point_node]
- for point_node in route_point_nodes.items
+ [float(str(subnode)) for subnode in point_node] for point_node in route_point_nodes.items
+ ]
+ event.routes = [
+ {"original_points": route_points},
]
- event.routes = [{'original_points': route_points}, ]
markers_js = es5(markers_text)
markers_details = defaultdict(dict)
@@ -110,77 +123,96 @@ async def get_config(app, settings, args):
if isinstance(node, ExprStatement):
if isinstance(node.expr, Assign):
id = node.expr.left.value
- if id.startswith('markercp'):
- markers_details[id]['location'] = [float(str(loc_node)) for loc_node in node.expr.right.identifier.node.args.items[0].items]
+ if id.startswith("markercp"):
+ markers_details[id]["location"] = [
+ float(str(loc_node))
+ for loc_node in node.expr.right.identifier.node.args.items[0].items
+ ]
if isinstance(node.expr, FunctionCall):
id = node.expr.identifier.node.value
- if id.startswith('markercp') and node.expr.identifier.identifier.value == 'bindTooltip':
- markers_details[id]['text'] = node.expr.args.items[0].value[4:-5]
- config['markers'] = [
+ if (
+ id.startswith("markercp")
+ and node.expr.identifier.identifier.value == "bindTooltip"
+ ):
+ markers_details[id]["text"] = node.expr.args.items[0].value[4:-5]
+ config["markers"] = [
{
- 'marker_text': details['text'],
- 'position': {'lat': details['location'][0], 'lng': details['location'][1], },
+ "marker_text": details["text"],
+ "position": {
+ "lat": details["location"][0],
+ "lng": details["location"][1],
+ },
}
for details in markers_details.values()
]
- logger.info('Processing route.')
+ logger.info("Processing route.")
await process_route(settings, event.routes[0])
process_secondary_route_details(event.routes)
- logger.info('Saving.')
+ logger.info("Saving.")
update_bounds_inner(event)
assign_rider_colors_inner(event)
- await event.save(f'{event_name}: load config from trackleaders', tree_writer=tree_writer, save_routes=True)
+ await event.save(
+ f"{event_name}: load config from trackleaders",
+ tree_writer=tree_writer,
+ save_routes=True,
+ )
async def start_event_tracker(app, event, rider_name, tracker_data, start, end):
- return await start_tracker(app, rider_name, tracker_data['name'], tracker_data['event'], end)
+ return await start_tracker(app, rider_name, tracker_data["name"], tracker_data["event"], end)
async def start_tracker(app, tracker_name, name, event, end):
- tracker = Tracker(f'trackleaders.{tracker_name}')
+ tracker = Tracker(f"trackleaders.{tracker_name}")
monitor_task = asyncio.ensure_future(monitor_feed(app, tracker, name, event, end))
tracker.stop = monitor_task.cancel
tracker.completed = monitor_task
return tracker
-recived_at_re = re.compile('received at: (.*?)
')
+
+recived_at_re = re.compile("received at: (.*?)
")
def datetime_parse(str):
dt_parse_tzinfos = {
"BST": 3600,
"CET": 7200,
- "SAST": dateutil.tz.gettz('Africa/Johannesburg'),
+ "SAST": dateutil.tz.gettz("Africa/Johannesburg"),
}
- brackets_removed = str.replace('(', '').replace(')', '')
+ brackets_removed = str.replace("(", "").replace(")", "")
try:
localized = dateutil.parser.parse(brackets_removed, tzinfos=dt_parse_tzinfos)
except ValueError as e:
- raise ValueError('{}: {}'.format(str(e), brackets_removed))
+ raise ValueError("{}: {}".format(str(e), brackets_removed))
return localized.replace(tzinfo=None)
-position_re = re.compile(r'imarker(?P\d+) = L.marker\(\[(?P.+?),(?P.+?)\]')
-time_re = re.compile(r'imarker(?P\d+)\.bindPopup\(\'.+
Point #\d+? received at: (?P