Skip to content

Commit c981c40

Browse files
authored
BCK-7387 Update firestarter with diversion handling (#44)
* Added check for diversions. * Perform diversion check on flightplan and flifo messages. * Update tests with a diversion and add diverted column * Look for new flights that haven't had ETMS messages yet.
1 parent 0bd832d commit c981c40

File tree

2 files changed

+48
-1
lines changed

2 files changed

+48
-1
lines changed

db-updater/main.py

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,7 @@
8686
sa.Column("predicted_off", TIMESTAMP_TZ()),
8787
sa.Column("predicted_on", TIMESTAMP_TZ()),
8888
sa.Column("predicted_in", TIMESTAMP_TZ()),
89+
sa.Column("diverted", sa.Boolean),
8990
)
9091
VALID_EVENTS = {"arrival", "cancellation", "departure", "flightplan", "onblock", "offblock", "extendedFlightInfo", "flifo"}
9192
elif TABLE == "positions":
@@ -188,6 +189,7 @@
188189
cache_lock = threading.Lock()
189190
SQLITE_VAR_LIMIT = None
190191

192+
dest_history = dict()
191193

192194
class Cache(ABC):
193195
"""A cache for accumulating flight or position information which can be flushed as necessary."""
@@ -368,6 +370,7 @@ def chunk(values: Iterable, chunk_size: Optional[int]) -> Iterator:
368370

369371
def add_to_cache(data: dict) -> None:
370372
"""add entry to the cache"""
373+
check_for_new_destination(data)
371374
converted = convert_msg_fields(data)
372375
with cache_lock:
373376
cache.add(converted)
@@ -409,6 +412,8 @@ def process_unknown_message(data: dict) -> None:
409412

410413
def process_arrival_message(data: dict) -> None:
411414
"""Arrival message type"""
415+
if "id" in data:
416+
clear_dest_history(data["id"])
412417
return add_to_cache(data)
413418

414419

@@ -432,12 +437,16 @@ def process_offblock_message(data: dict) -> None:
432437

433438
def process_onblock_message(data: dict) -> None:
434439
"""Onblock message type"""
440+
if "id" in data:
441+
clear_dest_history(data["id"])
435442
data["actual_in"] = data["clock"]
436443
return add_to_cache(data)
437444

438445

439446
def process_flifo_message(data: dict) -> None:
440447
"""flifo message type"""
448+
# flightplan and flifo messages may contain a new dest indicating a diversion, flag it if so
449+
check_for_diversions(data)
441450
# flifo messages try to help us with saner names, but we already convert
442451
# field names at the sqlalchemy level, so we actually need to convert the
443452
# nice names to ugly names so they can be converted again later...
@@ -465,6 +474,8 @@ def process_extended_flight_info_message(data: dict) -> None:
465474

466475
def process_flightplan_message(data: dict) -> None:
467476
"""Flightplan message type"""
477+
# flightplan and flifo messages may contain a new dest indicating a diversion, flag it if so
478+
check_for_diversions(data)
468479
disambiguate_altitude(data)
469480
return add_to_cache(data)
470481

@@ -480,6 +491,41 @@ def process_keepalive_message(data: dict) -> None:
480491
print(f'Based on keepalive["pitr"], we are {behind} behind realtime')
481492

482493

494+
def clear_dest_history(flight) -> None:
495+
"""Remove id from dest_history"""
496+
global dest_history
497+
if flight in dest_history:
498+
dest_history.pop(flight)
499+
return
500+
501+
502+
def check_for_diversions(data: dict) -> None:
503+
"""ETMS message, check destination"""
504+
global dest_history
505+
506+
dest = data.get("dest")
507+
flight = data.get("id")
508+
orig_dest = dest_history.get(flight)
509+
510+
if dest and flight:
511+
if orig_dest and orig_dest != dest:
512+
data["diverted"] = True
513+
514+
dest_history[flight] = dest
515+
516+
return
517+
518+
def check_for_new_destination(data: dict) -> None:
519+
"""Look for a new destination that hasn't been seen by an ETMS message yet"""
520+
global dest_history
521+
dest = data.get("dest")
522+
flight = data.get("id")
523+
524+
if dest and flight and flight not in dest_history:
525+
dest_history[flight] = dest
526+
527+
return
528+
483529
def disambiguate_altitude(data: dict):
484530
"""Replaces the alt field in the passed dict with an unambiguous field name"""
485531

db-updater/test/test_db_updater.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ class TestInsertAndExpire(unittest.TestCase):
3030
b'{"pitr": "1589549554", "type": "extendedFlightInfo", "ident": "SKW3284", "actual_departure_gate": "G11", "actual_departure_terminal": "3", "actual_out": "1589547780", "estimated_arrival_gate": "C3", "estimated_in": "1589553900", "estimated_out": "1589548200", "facility_hash": "F44B2C6C456D33FB", "facility_name": "Airline", "id": "SKW3284-1589345159-airline-0376", "scheduled_departure_terminal": "3", "scheduled_in": "1589554920", "scheduled_out": "1589548200"}',
3131
b'{"pitr": "1589551845", "type": "offblock", "ident": "UAL2465", "clock": "1589551845", "dest": "KDEN", "facility_hash": "23D67E4254EC60CD", "facility_name": "United Airlines", "id": "UAL2465-1589328337-fa-0001", "orig": "KORD"}',
3232
b'{"pitr": "1589551558", "type": "onblock", "ident": "CHH7691", "clock": "1589551547", "dest": "ZSOF", "facility_hash": "F44B2C6C456D33FB", "facility_name": "Airline", "id": "CHH7691-1589345160-airline-0022", "orig": "ZJSY"}',
33+
b'{"pitr": "1589551880", "type": "flifo", "ident": "UAL2465", "clock": "1589551845", "dest": "KHOU", "facility_hash": "23D67E4254EC60CD", "facility_name": "United Airlines", "id": "UAL2465-1589328337-fa-0001", "orig": "KORD"}',
3334
]
3435
flight_msgs = [FakeMessage(msg) for msg in _flight_msgs]
3536

@@ -44,7 +45,7 @@ class TestInsertAndExpire(unittest.TestCase):
4445
]
4546
position_msgs = [FakeMessage(msg) for msg in _position_msgs]
4647

47-
expected_flights_results = [('CGEYQ-1589545563-3-1-67', 'CGEYQ', 'CGEYQ', None, None, 'CYZX', 'CYQI', 'C177', None, 112, None, 6000, None, 'CYZX..YZX..MUXEL..OMTIV..CYQI', 'F', None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, datetime.datetime(2020, 5, 15, 13, 30), datetime.datetime(2020, 5, 15, 14, 18, 58), None, datetime.datetime(2020, 5, 15, 13, 15), None, None, None, datetime.datetime(2020, 5, 15, 13, 30), datetime.datetime(2020, 5, 15, 14, 18, 58), None), ('BLOCKED-1589549420-adhoc-0', 'BLOCKED', 'BLOCKED', None, None, 'KIWA', None, 'C172', None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, datetime.datetime(2020, 5, 15, 13, 30, 20), None, None, None, None, None, None, None, None, None, None, None, None, None), ('BMJ64-1589543130-5-1-174', 'BMJ64', 'N104BA', None, None, 'KLJF', 'KBRD', None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, datetime.datetime(2020, 5, 15, 13, 27, 1), None, None, None, None, None, None, None, None, None, None, None, None), ('CES2187-1589431537-airline-0180', 'CES2187', None, None, None, 'ZLXY', 'ZSAM', 'A320', None, 299, None, None, True, None, 'X', None, None, None, None, None, None, None, None, None, None, None, datetime.datetime(2020, 5, 15, 14, 1, 26), None, None, None, None, None, datetime.datetime(2020, 5, 16, 5, 5), datetime.datetime(2020, 5, 16, 7, 40), None, datetime.datetime(2020, 5, 16, 5, 5), None, None, None, None, None, None), ('SKW3284-1589345159-airline-0376', 'SKW3284', None, None, None, None, None, None, None, None, None, None, None, None, None, None, 'C3', 'G11', None, None, None, '3', '3', None, None, None, None, datetime.datetime(2020, 5, 15, 13, 3), None, None, None, datetime.datetime(2020, 5, 15, 13, 10), None, None, datetime.datetime(2020, 5, 15, 14, 45), None, datetime.datetime(2020, 5, 15, 13, 10), datetime.datetime(2020, 5, 15, 15, 2), None, None, None, None), ('UAL2465-1589328337-fa-0001', 'UAL2465', None, None, None, 'KORD', 'KDEN', None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, datetime.datetime(2020, 5, 15, 14, 10, 45), None, None, None, None, None, None, None, None, None, None, None, None, None, None), ('CHH7691-1589345160-airline-0022', 'CHH7691', None, None, None, 'ZJSY', 'ZSOF', None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, datetime.datetime(2020, 5, 15, 14, 5, 47), None, None, None, None, None, None, None, None, None, None, None)]
48+
expected_flights_results = [('CGEYQ-1589545563-3-1-67', 'CGEYQ', 'CGEYQ', None, None, 'CYZX', 'CYQI', 'C177', None, 112, None, 6000, None, 'CYZX..YZX..MUXEL..OMTIV..CYQI', 'F', None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, datetime.datetime(2020, 5, 15, 13, 30), datetime.datetime(2020, 5, 15, 14, 18, 58), None, datetime.datetime(2020, 5, 15, 13, 15), None, None, None, datetime.datetime(2020, 5, 15, 13, 30), datetime.datetime(2020, 5, 15, 14, 18, 58), None, None), ('BLOCKED-1589549420-adhoc-0', 'BLOCKED', 'BLOCKED', None, None, 'KIWA', None, 'C172', None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, datetime.datetime(2020, 5, 15, 13, 30, 20), None, None, None, None, None, None, None, None, None, None, None, None, None, None), ('BMJ64-1589543130-5-1-174', 'BMJ64', 'N104BA', None, None, 'KLJF', 'KBRD', None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, datetime.datetime(2020, 5, 15, 13, 27, 1), None, None, None, None, None, None, None, None, None, None, None, None, None), ('CES2187-1589431537-airline-0180', 'CES2187', None, None, None, 'ZLXY', 'ZSAM', 'A320', None, 299, None, None, True, None, 'X', None, None, None, None, None, None, None, None, None, None, None, datetime.datetime(2020, 5, 15, 14, 1, 26), None, None, None, None, None, datetime.datetime(2020, 5, 16, 5, 5), datetime.datetime(2020, 5, 16, 7, 40), None, datetime.datetime(2020, 5, 16, 5, 5), None, None, None, None, None, None, None), ('SKW3284-1589345159-airline-0376', 'SKW3284', None, None, None, None, None, None, None, None, None, None, None, None, None, None, 'C3', 'G11', None, None, None, '3', '3', None, None, None, None, datetime.datetime(2020, 5, 15, 13, 3), None, None, None, datetime.datetime(2020, 5, 15, 13, 10), None, None, datetime.datetime(2020, 5, 15, 14, 45), None, datetime.datetime(2020, 5, 15, 13, 10), datetime.datetime(2020, 5, 15, 15, 2), None, None, None, None, None), ('UAL2465-1589328337-fa-0001', 'UAL2465', None, None, None, 'KORD', 'KHOU', None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, datetime.datetime(2020, 5, 15, 14, 10, 45), None, None, None, None, None, None, None, None, None, None, None, None, None, None, 1), ('CHH7691-1589345160-airline-0022', 'CHH7691', None, None, None, 'ZJSY', 'ZSOF', None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, datetime.datetime(2020, 5, 15, 14, 5, 47), None, None, None, None, None, None, None, None, None, None, None, None)]
4849
expected_positions_results = [('UEA2236-1591584600-schedule-0391', datetime.datetime(2020, 6, 10, 4, 36, 7), '29.84762', '104.34529', 'E4030EF8218DE21D', 'FlightAware ADS-B', 'A', None, None, 13800, None, ' ', None, 'B9985', 'ZGGG', 'ZUUU', None, None, None, None, 319, '336', '336.1', None, '780BBB', '0.484', None, None, None, None, None, None, None, None, None, None, None, None, 600, None, 250, 314, 4525, 4, 1, None, None, None, None, None, None), ('CES5489-1591589400-schedule-0472', datetime.datetime(2020, 6, 10, 4, 36, 3), '31.43910', '115.68351', '18249C69E5BC3F1B', 'FlightAware ADS-B', 'A', None, None, 17700, None, ' ', None, 'B1610', 'ZSOF', 'ZPPP', None, None, None, None, 401, '244', '250.0', None, '780D1B', '0.648', None, None, None, None, None, None, None, None, None, None, None, None, 512, None, 312, 414, 5005, -4, 1, None, None, None, None, None, None), ('CAL782-1591588500-schedule-0187', datetime.datetime(2020, 6, 10, 4, 36, 20), '12.81155', '108.19659', '52446C6CE8E0B859', 'FlightAware ADS-B', 'A', None, None, 34500, None, 'C', None, 'B18909', 'VVTS', 'RCTP', None, None, None, None, 494, '36', '37.3', None, '899128', '0.852', None, None, None, None, None, None, None, None, None, None, None, None, 244, None, 297, 510, 5620, -37, 1, None, None, None, None, None, None), ('CWA926-1591760300-2-0-98', datetime.datetime(2020, 6, 10, 4, 36, 30), '57.87987', '-116.59763', 'B566E7DB03CCFF2C', 'FlightAware ADS-B', 'A', None, None, 23575, None, 'C', None, 'CGLUQ', 'CYOJ', 'CYEG', None, None, None, None, 234, '158', None, None, 'C063CD', None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, 5213, None, None, None, None, None, None, None, None), ('SOO7930-1591724581-7-3-192', datetime.datetime(2020, 6, 10, 4, 36, 49), '64.62843', '-143.16364', 'C3DCCEF85E56D79D', 'FlightAware ADS-B', 'A', None, 'B77L', 34000, None, ' ', None, 'N702GT', 'KJFK', 'RKSI', None, None, None, None, 494, '272', None, None, 'A95CA4', None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, 3064, None, None, None, None, None, None, None, None), ('CES5489-1591589400-schedule-0472', datetime.datetime(2020, 6, 10, 4, 36, 59), '29.50584', '106.56178', 'E4030EF8218DE21D', 'FlightAware ADS-B', 'A', None, None, 38100, None, ' ', None, 'B1610', 'ZSOF', 'ZUUU', None, None, None, None, 464, '179', '184.9', None, '780D1B', '0.804', None, None, None, None, None, None, None, None, None, None, None, None, 205, None, 255, 470, 4113, -48, 1, None, None, None, None, None, None), ('N517MT-1591762834-adhoc-0', datetime.datetime(2020, 6, 10, 4, 37, 8), '33.58891', '-97.21146', 'A4C87376ACECA2CE', 'FlightAware ADS-B', 'A', None, 'EC35', 1550, None, 'D', None, 'N517MT', 'KDTO', None, None, None, None, None, 119, '13', None, None, 'A67C95', None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, 1200, None, None, None, None, None, None, None, None)]
4950
# fmt: on
5051

0 commit comments

Comments
 (0)