Can now use basestation.sqb for aircraft type info.

This commit is contained in:
John Wiseman 2019-12-17 11:01:35 -08:00
parent 6d80d0e77c
commit 8cc08d0ebc
3 changed files with 167 additions and 8 deletions

141
cleanbasestation.py Normal file
View File

@ -0,0 +1,141 @@
import multiprocessing
import os
import re
import shutil
import sqlite3
import sys
import time
def error(msg, *args):
sys.stderr.write((msg % args) + '\n')
sys.stderr.flush()
def print_usage():
print('%s: <src.sqb> <dest.sqb>')
def process_changes(db_path, queue):
db = sqlite3.connect(db_path, isolation_level=None)
# db.execute('pragma journal_mode=wal')
db.execute('BEGIN')
rec = queue.get()
num_executes = 0
start_time = time.time()
while rec != 'DONE':
sql = 'UPDATE Aircraft SET Type = ?, RegisteredOwners = ? where ModeS = ?'
db.execute(sql, (rec['Type'], rec['RegisteredOwners'], rec['ModeS']))
num_executes += 1
if num_executes == 10000:
db.commit()
end_time = time.time()
print('Writer: Processing %.1f records/sec' % (num_executes / (end_time - start_time)))
db.execute('BEGIN')
num_executes = 0
start_time = time.time()
rec = queue.get()
print('Writer: Finished')
db.commit()
db.close()
CITY_STATE_CLEAN_RE = re.compile(r' +- +[a-zA-Z0-9 ]+, [A-Za-z]{2}$')
TITLE_CASE = [
'AIR',
'CO',
'OF',
'AND',
'INC',
'ONE',
'TWO',
'FLI',
'HI',
'SAN'
]
NOT_TITLE_CASE = [
'TIS-B'
]
TITLE_CASE_EXCEPTION_RE = re.compile('[0-9]')
def title_case(s):
if (TITLE_CASE_EXCEPTION_RE.search(s) or
s in NOT_TITLE_CASE or
(len(s) <= 3 and s not in TITLE_CASE)):
return s
else:
return s.title()
def fix_type(s):
if s is not None:
tokens = [p for p in s.split(' ') if p]
tokens = [title_case(t) for t in tokens]
s = ' '.join(tokens)
return s
def fix_registered_owners(s):
if s is not None:
# Remove " - <city>, <state abbrev>' suffix.
s = CITY_STATE_CLEAN_RE.sub('', s)
# Coalesce multiple spaces.
tokens = [p for p in s.split(' ') if p]
tokens = [title_case(t) for t in tokens]
s = ' '.join(tokens)
return s
def fix_db(db_in_path, db_out_path):
queue = multiprocessing.Queue()
worker = multiprocessing.Process(
target=process_changes,
args=(db_out_path, queue))
db = sqlite3.connect(db_in_path)
db.row_factory = sqlite3.Row
worker.start()
num_records = 0
num_fixed_records = 0
for row in db.execute('select * from Aircraft'):
num_records += 1
ac_type = row['Type']
ac_regd_owners = row['RegisteredOwners']
new_ac_type = fix_type(ac_type)
# new_ac_regd_owners = fix_registered_owners(ac_regd_owners)
new_ac_regd_owners = ac_regd_owners
if new_ac_type != ac_type or new_ac_regd_owners != ac_regd_owners:
# print('%s -> %s' % (ac_type, new_ac_type))
num_fixed_records += 1
queue.put({'ModeS': row['ModeS'],
'Type': new_ac_type,
'RegisteredOwners': new_ac_regd_owners},
True,
10)
if num_records % 10000 == 0:
print('Reader: Processed %s records, fixed %s' % (num_records, num_fixed_records))
queue.put('DONE')
print('Reader: Finished')
worker.join()
db.close()
def main(args):
if len(args) != 2:
print_usage()
sys.exit(1)
db_in = args[0]
db_out = args[1]
try:
print('Copying %s to %s...' % (db_in, db_out))
shutil.copyfile(db_in, db_out)
fix_db(db_in, db_out)
except:
#os.remove(db_out)
raise
if __name__ == '__main__':
main(sys.argv[1:])

View File

@ -12,6 +12,7 @@
"puppeteer": "^2.0.0", "puppeteer": "^2.0.0",
"request": "^2.88.0", "request": "^2.88.0",
"request-promise-native": "^1.0.8", "request-promise-native": "^1.0.8",
"sqlite": "^3.0.3",
"twit": "^2.2.11", "twit": "^2.2.11",
"winston": "^3.2.1" "winston": "^3.2.1"
} }

View File

@ -14,11 +14,21 @@
[lemondronor.circlebot.logging :as logging] [lemondronor.circlebot.logging :as logging]
[lemondronor.circlebot.pelias :as pelias] [lemondronor.circlebot.pelias :as pelias]
[lemondronor.circlebot.twitter :as twitter] [lemondronor.circlebot.twitter :as twitter]
[lemondronor.circlebot.util :as util])) [lemondronor.circlebot.util :as util]
["sqlite" :as sqlite]))
(logging/deflog "circlebot" logger) (logging/deflog "circlebot" logger)
(defn get-basestation-sqb-record [icao db-path]
(log-info "%s: Looking up in %s" icao db-path)
(p/let [record
(p/-> (sqlite/open db-path clj->js { js/Promise js/Promise })
(.get "SELECT Registration, Type, RegisteredOwners from Aircraft where ModeS = ?" icao))]
(log-info "%s: basestation.sqb: %s" icao (js->clj record :keywordize-keys true))
(js->clj record :keywordize-keys true)))
(defn parse-adsbexchange-ac-element [e] (defn parse-adsbexchange-ac-element [e]
(let [nilstr #(if (= % "") nil %) (let [nilstr #(if (= % "") nil %)
numstr #(if (= % "") nil (js/parseFloat %))] numstr #(if (= % "") nil (js/parseFloat %))]
@ -263,7 +273,8 @@
(def description-templates (def description-templates
(map generation/parse-template (map generation/parse-template
[(str "[{registration}|{militaryregistration}, a military aircraft,|" [(str "[{registration}|{registration}, a {type},|{militaryregistration}, a military aircraft,|"
"{militaryregistration}, a military {type},|"
"Aircraft with unknown registration, ICAO {icao}|" "Aircraft with unknown registration, ICAO {icao}|"
"Military aircraft with unknown registration, ICAO {militaryicao}] " "Military aircraft with unknown registration, ICAO {militaryicao}] "
"?:[(callsign {callsign}) ]" "?:[(callsign {callsign}) ]"
@ -295,11 +306,13 @@
(defn to-fixed [n d] (defn to-fixed [n d]
(.toFixed n d)) (.toFixed n d))
(defn generate-description [ac reverse wiki-nearby nearby] (defn generate-description [ac sqb reverse wiki-nearby nearby]
(let [rev-props (:properties reverse) (let [rev-props (:properties reverse)
nearby (:properties (first nearby)) nearby (:properties (first nearby))
wiki-nearby (:properties (first wiki-nearby)) wiki-nearby (:properties (first wiki-nearby))
info (cond-> (-> ac (dissoc :history) (merge rev-props)) info (cond-> (-> ac (dissoc :history :type) (merge rev-props))
(:Type sqb)
(assoc :type (:Type sqb))
(:military? ac) (:military? ac)
(-> (assoc :militaryregistration (:registration ac) (-> (assoc :militaryregistration (:registration ac)
:militaryicao (:icao ac))) :militaryicao (:icao ac)))
@ -338,7 +351,6 @@
(defn process-potential-circle [ac config now] (defn process-potential-circle [ac config now]
(p/let [icao (:icao ac) (p/let [icao (:icao ac)
recent-positions (recent-history (:history ac)) recent-positions (recent-history (:history ac))
_ (log-info "WOO %s" (last recent-positions))
_ (log-info "%s: Recent history has %s positions, most recent is %s secs old" _ (log-info "%s: Recent history has %s positions, most recent is %s secs old"
icao icao
(count recent-positions) (count recent-positions)
@ -370,7 +382,9 @@
:layers "venue" :layers "venue"
:size 50}) :size 50})
nearby (:features nearby) nearby (:features nearby)
wiki-nearby (filter feature-has-wikipedia-page? nearby)] wiki-nearby (filter feature-has-wikipedia-page? nearby)
sqb (if-let [sqb-path (:basestation-sqb config)]
(get-basestation-sqb-record icao sqb-path))]
(log-info "%s: Nearby geo search: %s potential landmarks, %s with wikipedia pages" (log-info "%s: Nearby geo search: %s potential landmarks, %s with wikipedia pages"
icao (count nearby) (count wiki-nearby)) icao (count nearby) (count wiki-nearby))
(log-info "%s" (->> nearby (take 3) (map :properties))) (log-info "%s" (->> nearby (take 3) (map :properties)))
@ -385,7 +399,7 @@
icao icao
(get-in f [:properties :label] f) (get-in f [:properties :label] f)
(get-in f [:properties :addendum] f))) (get-in f [:properties :addendum] f)))
(let [description (generate-description ac coarse wiki-nearby nearby)] (let [description (generate-description ac sqb coarse wiki-nearby nearby)]
(log-warn "Description: %s" description) (log-warn "Description: %s" description)
description))]) description))])
(fn [[image-path description]] (fn [[image-path description]]
@ -421,13 +435,16 @@
(.requiredOption "--lon <lat>" "Longitude of the circle of the region of interest" parse-number) (.requiredOption "--lon <lat>" "Longitude of the circle of the region of interest" parse-number)
(.requiredOption "--url <url>" "API url") (.requiredOption "--url <url>" "API url")
(.option "--radius <radius>" "Radius of the circle of interest, in nautical miles" 20 parse-number) (.option "--radius <radius>" "Radius of the circle of interest, in nautical miles" 20 parse-number)
(.option "--basestation-sqb <path>" "Path to a basestation.sqb database file")
(.option "--no-tweeting" "Do not tweet.") (.option "--no-tweeting" "Do not tweet.")
(.parse (.-argv js/process))) (.parse (.-argv js/process)))
(let [start-time (current-time)] (let [start-time (current-time)]
(p/then (p/all [(read-history-db history-db-path) (p/then (p/all [(read-history-db history-db-path)
(util/read-config secrets-path)]) (util/read-config secrets-path)])
(fn [[db config]] (fn [[db config]]
(p/let [config (assoc config :tweeting-enabled? (.-tweeting commander)) (p/let [config (assoc config
:tweeting-enabled? (.-tweeting commander)
:basestation-sqb (.-basestationSqb commander))
data (get-adsbexchange-live-data data (get-adsbexchange-live-data
{:url (.-url commander) {:url (.-url commander)
:api-key (get-in config [:adsbx :api-key]) :api-key (get-in config [:adsbx :api-key])