From 8cc08d0ebc320c3797c1a885a7e36199697408a0 Mon Sep 17 00:00:00 2001 From: John Wiseman Date: Tue, 17 Dec 2019 11:01:35 -0800 Subject: [PATCH] Can now use basestation.sqb for aircraft type info. --- cleanbasestation.py | 141 ++++++++++++++++++++++++++++ package.json | 1 + src/main/lemondronor/circlebot.cljs | 33 +++++-- 3 files changed, 167 insertions(+), 8 deletions(-) create mode 100644 cleanbasestation.py diff --git a/cleanbasestation.py b/cleanbasestation.py new file mode 100644 index 0000000..d402933 --- /dev/null +++ b/cleanbasestation.py @@ -0,0 +1,141 @@ +import multiprocessing +import os +import re +import shutil +import sqlite3 +import sys +import time + + +def error(msg, *args): + sys.stderr.write((msg % args) + '\n') + sys.stderr.flush() + + +def print_usage(): + print('%s: ') + + +def process_changes(db_path, queue): + db = sqlite3.connect(db_path, isolation_level=None) + # db.execute('pragma journal_mode=wal') + db.execute('BEGIN') + rec = queue.get() + num_executes = 0 + start_time = time.time() + while rec != 'DONE': + sql = 'UPDATE Aircraft SET Type = ?, RegisteredOwners = ? where ModeS = ?' + db.execute(sql, (rec['Type'], rec['RegisteredOwners'], rec['ModeS'])) + num_executes += 1 + if num_executes == 10000: + db.commit() + end_time = time.time() + print('Writer: Processing %.1f records/sec' % (num_executes / (end_time - start_time))) + db.execute('BEGIN') + num_executes = 0 + start_time = time.time() + rec = queue.get() + print('Writer: Finished') + db.commit() + db.close() + + +CITY_STATE_CLEAN_RE = re.compile(r' +- +[a-zA-Z0-9 ]+, [A-Za-z]{2}$') + +TITLE_CASE = [ + 'AIR', + 'CO', + 'OF', + 'AND', + 'INC', + 'ONE', + 'TWO', + 'FLI', + 'HI', + 'SAN' +] + +NOT_TITLE_CASE = [ + 'TIS-B' +] + + +TITLE_CASE_EXCEPTION_RE = re.compile('[0-9]') + +def title_case(s): + if (TITLE_CASE_EXCEPTION_RE.search(s) or + s in NOT_TITLE_CASE or + (len(s) <= 3 and s not in TITLE_CASE)): + return s + else: + return s.title() + +def fix_type(s): + if s is not None: + tokens = [p for p in s.split(' ') if p] + tokens = [title_case(t) for t in tokens] + s = ' '.join(tokens) + return s + + +def fix_registered_owners(s): + if s is not None: + # Remove " - , ' suffix. + s = CITY_STATE_CLEAN_RE.sub('', s) + # Coalesce multiple spaces. + tokens = [p for p in s.split(' ') if p] + tokens = [title_case(t) for t in tokens] + s = ' '.join(tokens) + return s + + +def fix_db(db_in_path, db_out_path): + queue = multiprocessing.Queue() + worker = multiprocessing.Process( + target=process_changes, + args=(db_out_path, queue)) + db = sqlite3.connect(db_in_path) + db.row_factory = sqlite3.Row + worker.start() + num_records = 0 + num_fixed_records = 0 + for row in db.execute('select * from Aircraft'): + num_records += 1 + ac_type = row['Type'] + ac_regd_owners = row['RegisteredOwners'] + new_ac_type = fix_type(ac_type) + # new_ac_regd_owners = fix_registered_owners(ac_regd_owners) + new_ac_regd_owners = ac_regd_owners + if new_ac_type != ac_type or new_ac_regd_owners != ac_regd_owners: + # print('%s -> %s' % (ac_type, new_ac_type)) + num_fixed_records += 1 + queue.put({'ModeS': row['ModeS'], + 'Type': new_ac_type, + 'RegisteredOwners': new_ac_regd_owners}, + True, + 10) + if num_records % 10000 == 0: + print('Reader: Processed %s records, fixed %s' % (num_records, num_fixed_records)) + queue.put('DONE') + print('Reader: Finished') + worker.join() + db.close() + + +def main(args): + if len(args) != 2: + print_usage() + sys.exit(1) + db_in = args[0] + db_out = args[1] + try: + print('Copying %s to %s...' % (db_in, db_out)) + shutil.copyfile(db_in, db_out) + fix_db(db_in, db_out) + except: + #os.remove(db_out) + raise + + +if __name__ == '__main__': + main(sys.argv[1:]) diff --git a/package.json b/package.json index 868b6b9..4d74b4f 100644 --- a/package.json +++ b/package.json @@ -12,6 +12,7 @@ "puppeteer": "^2.0.0", "request": "^2.88.0", "request-promise-native": "^1.0.8", + "sqlite": "^3.0.3", "twit": "^2.2.11", "winston": "^3.2.1" } diff --git a/src/main/lemondronor/circlebot.cljs b/src/main/lemondronor/circlebot.cljs index f78fa94..9ce26e4 100644 --- a/src/main/lemondronor/circlebot.cljs +++ b/src/main/lemondronor/circlebot.cljs @@ -14,11 +14,21 @@ [lemondronor.circlebot.logging :as logging] [lemondronor.circlebot.pelias :as pelias] [lemondronor.circlebot.twitter :as twitter] - [lemondronor.circlebot.util :as util])) + [lemondronor.circlebot.util :as util] + ["sqlite" :as sqlite])) (logging/deflog "circlebot" logger) +(defn get-basestation-sqb-record [icao db-path] + (log-info "%s: Looking up in %s" icao db-path) + (p/let [record + (p/-> (sqlite/open db-path clj->js { js/Promise js/Promise }) + (.get "SELECT Registration, Type, RegisteredOwners from Aircraft where ModeS = ?" icao))] + (log-info "%s: basestation.sqb: %s" icao (js->clj record :keywordize-keys true)) + (js->clj record :keywordize-keys true))) + + (defn parse-adsbexchange-ac-element [e] (let [nilstr #(if (= % "") nil %) numstr #(if (= % "") nil (js/parseFloat %))] @@ -263,7 +273,8 @@ (def description-templates (map generation/parse-template - [(str "[{registration}|{militaryregistration}, a military aircraft,|" + [(str "[{registration}|{registration}, a {type},|{militaryregistration}, a military aircraft,|" + "{militaryregistration}, a military {type},|" "Aircraft with unknown registration, ICAO {icao}|" "Military aircraft with unknown registration, ICAO {militaryicao}] " "?:[(callsign {callsign}) ]" @@ -295,11 +306,13 @@ (defn to-fixed [n d] (.toFixed n d)) -(defn generate-description [ac reverse wiki-nearby nearby] +(defn generate-description [ac sqb reverse wiki-nearby nearby] (let [rev-props (:properties reverse) nearby (:properties (first nearby)) wiki-nearby (:properties (first wiki-nearby)) - info (cond-> (-> ac (dissoc :history) (merge rev-props)) + info (cond-> (-> ac (dissoc :history :type) (merge rev-props)) + (:Type sqb) + (assoc :type (:Type sqb)) (:military? ac) (-> (assoc :militaryregistration (:registration ac) :militaryicao (:icao ac))) @@ -338,7 +351,6 @@ (defn process-potential-circle [ac config now] (p/let [icao (:icao ac) recent-positions (recent-history (:history ac)) - _ (log-info "WOO %s" (last recent-positions)) _ (log-info "%s: Recent history has %s positions, most recent is %s secs old" icao (count recent-positions) @@ -370,7 +382,9 @@ :layers "venue" :size 50}) nearby (:features nearby) - wiki-nearby (filter feature-has-wikipedia-page? nearby)] + wiki-nearby (filter feature-has-wikipedia-page? nearby) + sqb (if-let [sqb-path (:basestation-sqb config)] + (get-basestation-sqb-record icao sqb-path))] (log-info "%s: Nearby geo search: %s potential landmarks, %s with wikipedia pages" icao (count nearby) (count wiki-nearby)) (log-info "%s" (->> nearby (take 3) (map :properties))) @@ -385,7 +399,7 @@ icao (get-in f [:properties :label] f) (get-in f [:properties :addendum] f))) - (let [description (generate-description ac coarse wiki-nearby nearby)] + (let [description (generate-description ac sqb coarse wiki-nearby nearby)] (log-warn "Description: %s" description) description))]) (fn [[image-path description]] @@ -421,13 +435,16 @@ (.requiredOption "--lon " "Longitude of the circle of the region of interest" parse-number) (.requiredOption "--url " "API url") (.option "--radius " "Radius of the circle of interest, in nautical miles" 20 parse-number) + (.option "--basestation-sqb " "Path to a basestation.sqb database file") (.option "--no-tweeting" "Do not tweet.") (.parse (.-argv js/process))) (let [start-time (current-time)] (p/then (p/all [(read-history-db history-db-path) (util/read-config secrets-path)]) (fn [[db config]] - (p/let [config (assoc config :tweeting-enabled? (.-tweeting commander)) + (p/let [config (assoc config + :tweeting-enabled? (.-tweeting commander) + :basestation-sqb (.-basestationSqb commander)) data (get-adsbexchange-live-data {:url (.-url commander) :api-key (get-in config [:adsbx :api-key])