Can now use basestation.sqb for aircraft type info.
This commit is contained in:
parent
6d80d0e77c
commit
8cc08d0ebc
141
cleanbasestation.py
Normal file
141
cleanbasestation.py
Normal file
@ -0,0 +1,141 @@
|
|||||||
|
import multiprocessing
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import shutil
|
||||||
|
import sqlite3
|
||||||
|
import sys
|
||||||
|
import time
|
||||||
|
|
||||||
|
|
||||||
|
def error(msg, *args):
|
||||||
|
sys.stderr.write((msg % args) + '\n')
|
||||||
|
sys.stderr.flush()
|
||||||
|
|
||||||
|
|
||||||
|
def print_usage():
|
||||||
|
print('%s: <src.sqb> <dest.sqb>')
|
||||||
|
|
||||||
|
|
||||||
|
def process_changes(db_path, queue):
|
||||||
|
db = sqlite3.connect(db_path, isolation_level=None)
|
||||||
|
# db.execute('pragma journal_mode=wal')
|
||||||
|
db.execute('BEGIN')
|
||||||
|
rec = queue.get()
|
||||||
|
num_executes = 0
|
||||||
|
start_time = time.time()
|
||||||
|
while rec != 'DONE':
|
||||||
|
sql = 'UPDATE Aircraft SET Type = ?, RegisteredOwners = ? where ModeS = ?'
|
||||||
|
db.execute(sql, (rec['Type'], rec['RegisteredOwners'], rec['ModeS']))
|
||||||
|
num_executes += 1
|
||||||
|
if num_executes == 10000:
|
||||||
|
db.commit()
|
||||||
|
end_time = time.time()
|
||||||
|
print('Writer: Processing %.1f records/sec' % (num_executes / (end_time - start_time)))
|
||||||
|
db.execute('BEGIN')
|
||||||
|
num_executes = 0
|
||||||
|
start_time = time.time()
|
||||||
|
rec = queue.get()
|
||||||
|
print('Writer: Finished')
|
||||||
|
db.commit()
|
||||||
|
db.close()
|
||||||
|
|
||||||
|
|
||||||
|
CITY_STATE_CLEAN_RE = re.compile(r' +- +[a-zA-Z0-9 ]+, [A-Za-z]{2}$')
|
||||||
|
|
||||||
|
TITLE_CASE = [
|
||||||
|
'AIR',
|
||||||
|
'CO',
|
||||||
|
'OF',
|
||||||
|
'AND',
|
||||||
|
'INC',
|
||||||
|
'ONE',
|
||||||
|
'TWO',
|
||||||
|
'FLI',
|
||||||
|
'HI',
|
||||||
|
'SAN'
|
||||||
|
]
|
||||||
|
|
||||||
|
NOT_TITLE_CASE = [
|
||||||
|
'TIS-B'
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
TITLE_CASE_EXCEPTION_RE = re.compile('[0-9]')
|
||||||
|
|
||||||
|
def title_case(s):
|
||||||
|
if (TITLE_CASE_EXCEPTION_RE.search(s) or
|
||||||
|
s in NOT_TITLE_CASE or
|
||||||
|
(len(s) <= 3 and s not in TITLE_CASE)):
|
||||||
|
return s
|
||||||
|
else:
|
||||||
|
return s.title()
|
||||||
|
|
||||||
|
def fix_type(s):
|
||||||
|
if s is not None:
|
||||||
|
tokens = [p for p in s.split(' ') if p]
|
||||||
|
tokens = [title_case(t) for t in tokens]
|
||||||
|
s = ' '.join(tokens)
|
||||||
|
return s
|
||||||
|
|
||||||
|
|
||||||
|
def fix_registered_owners(s):
|
||||||
|
if s is not None:
|
||||||
|
# Remove " - <city>, <state abbrev>' suffix.
|
||||||
|
s = CITY_STATE_CLEAN_RE.sub('', s)
|
||||||
|
# Coalesce multiple spaces.
|
||||||
|
tokens = [p for p in s.split(' ') if p]
|
||||||
|
tokens = [title_case(t) for t in tokens]
|
||||||
|
s = ' '.join(tokens)
|
||||||
|
return s
|
||||||
|
|
||||||
|
|
||||||
|
def fix_db(db_in_path, db_out_path):
|
||||||
|
queue = multiprocessing.Queue()
|
||||||
|
worker = multiprocessing.Process(
|
||||||
|
target=process_changes,
|
||||||
|
args=(db_out_path, queue))
|
||||||
|
db = sqlite3.connect(db_in_path)
|
||||||
|
db.row_factory = sqlite3.Row
|
||||||
|
worker.start()
|
||||||
|
num_records = 0
|
||||||
|
num_fixed_records = 0
|
||||||
|
for row in db.execute('select * from Aircraft'):
|
||||||
|
num_records += 1
|
||||||
|
ac_type = row['Type']
|
||||||
|
ac_regd_owners = row['RegisteredOwners']
|
||||||
|
new_ac_type = fix_type(ac_type)
|
||||||
|
# new_ac_regd_owners = fix_registered_owners(ac_regd_owners)
|
||||||
|
new_ac_regd_owners = ac_regd_owners
|
||||||
|
if new_ac_type != ac_type or new_ac_regd_owners != ac_regd_owners:
|
||||||
|
# print('%s -> %s' % (ac_type, new_ac_type))
|
||||||
|
num_fixed_records += 1
|
||||||
|
queue.put({'ModeS': row['ModeS'],
|
||||||
|
'Type': new_ac_type,
|
||||||
|
'RegisteredOwners': new_ac_regd_owners},
|
||||||
|
True,
|
||||||
|
10)
|
||||||
|
if num_records % 10000 == 0:
|
||||||
|
print('Reader: Processed %s records, fixed %s' % (num_records, num_fixed_records))
|
||||||
|
queue.put('DONE')
|
||||||
|
print('Reader: Finished')
|
||||||
|
worker.join()
|
||||||
|
db.close()
|
||||||
|
|
||||||
|
|
||||||
|
def main(args):
|
||||||
|
if len(args) != 2:
|
||||||
|
print_usage()
|
||||||
|
sys.exit(1)
|
||||||
|
db_in = args[0]
|
||||||
|
db_out = args[1]
|
||||||
|
try:
|
||||||
|
print('Copying %s to %s...' % (db_in, db_out))
|
||||||
|
shutil.copyfile(db_in, db_out)
|
||||||
|
fix_db(db_in, db_out)
|
||||||
|
except:
|
||||||
|
#os.remove(db_out)
|
||||||
|
raise
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
main(sys.argv[1:])
|
@ -12,6 +12,7 @@
|
|||||||
"puppeteer": "^2.0.0",
|
"puppeteer": "^2.0.0",
|
||||||
"request": "^2.88.0",
|
"request": "^2.88.0",
|
||||||
"request-promise-native": "^1.0.8",
|
"request-promise-native": "^1.0.8",
|
||||||
|
"sqlite": "^3.0.3",
|
||||||
"twit": "^2.2.11",
|
"twit": "^2.2.11",
|
||||||
"winston": "^3.2.1"
|
"winston": "^3.2.1"
|
||||||
}
|
}
|
||||||
|
@ -14,11 +14,21 @@
|
|||||||
[lemondronor.circlebot.logging :as logging]
|
[lemondronor.circlebot.logging :as logging]
|
||||||
[lemondronor.circlebot.pelias :as pelias]
|
[lemondronor.circlebot.pelias :as pelias]
|
||||||
[lemondronor.circlebot.twitter :as twitter]
|
[lemondronor.circlebot.twitter :as twitter]
|
||||||
[lemondronor.circlebot.util :as util]))
|
[lemondronor.circlebot.util :as util]
|
||||||
|
["sqlite" :as sqlite]))
|
||||||
|
|
||||||
(logging/deflog "circlebot" logger)
|
(logging/deflog "circlebot" logger)
|
||||||
|
|
||||||
|
|
||||||
|
(defn get-basestation-sqb-record [icao db-path]
|
||||||
|
(log-info "%s: Looking up in %s" icao db-path)
|
||||||
|
(p/let [record
|
||||||
|
(p/-> (sqlite/open db-path clj->js { js/Promise js/Promise })
|
||||||
|
(.get "SELECT Registration, Type, RegisteredOwners from Aircraft where ModeS = ?" icao))]
|
||||||
|
(log-info "%s: basestation.sqb: %s" icao (js->clj record :keywordize-keys true))
|
||||||
|
(js->clj record :keywordize-keys true)))
|
||||||
|
|
||||||
|
|
||||||
(defn parse-adsbexchange-ac-element [e]
|
(defn parse-adsbexchange-ac-element [e]
|
||||||
(let [nilstr #(if (= % "") nil %)
|
(let [nilstr #(if (= % "") nil %)
|
||||||
numstr #(if (= % "") nil (js/parseFloat %))]
|
numstr #(if (= % "") nil (js/parseFloat %))]
|
||||||
@ -263,7 +273,8 @@
|
|||||||
|
|
||||||
(def description-templates
|
(def description-templates
|
||||||
(map generation/parse-template
|
(map generation/parse-template
|
||||||
[(str "[{registration}|{militaryregistration}, a military aircraft,|"
|
[(str "[{registration}|{registration}, a {type},|{militaryregistration}, a military aircraft,|"
|
||||||
|
"{militaryregistration}, a military {type},|"
|
||||||
"Aircraft with unknown registration, ICAO {icao}|"
|
"Aircraft with unknown registration, ICAO {icao}|"
|
||||||
"Military aircraft with unknown registration, ICAO {militaryicao}] "
|
"Military aircraft with unknown registration, ICAO {militaryicao}] "
|
||||||
"?:[(callsign {callsign}) ]"
|
"?:[(callsign {callsign}) ]"
|
||||||
@ -295,11 +306,13 @@
|
|||||||
(defn to-fixed [n d]
|
(defn to-fixed [n d]
|
||||||
(.toFixed n d))
|
(.toFixed n d))
|
||||||
|
|
||||||
(defn generate-description [ac reverse wiki-nearby nearby]
|
(defn generate-description [ac sqb reverse wiki-nearby nearby]
|
||||||
(let [rev-props (:properties reverse)
|
(let [rev-props (:properties reverse)
|
||||||
nearby (:properties (first nearby))
|
nearby (:properties (first nearby))
|
||||||
wiki-nearby (:properties (first wiki-nearby))
|
wiki-nearby (:properties (first wiki-nearby))
|
||||||
info (cond-> (-> ac (dissoc :history) (merge rev-props))
|
info (cond-> (-> ac (dissoc :history :type) (merge rev-props))
|
||||||
|
(:Type sqb)
|
||||||
|
(assoc :type (:Type sqb))
|
||||||
(:military? ac)
|
(:military? ac)
|
||||||
(-> (assoc :militaryregistration (:registration ac)
|
(-> (assoc :militaryregistration (:registration ac)
|
||||||
:militaryicao (:icao ac)))
|
:militaryicao (:icao ac)))
|
||||||
@ -338,7 +351,6 @@
|
|||||||
(defn process-potential-circle [ac config now]
|
(defn process-potential-circle [ac config now]
|
||||||
(p/let [icao (:icao ac)
|
(p/let [icao (:icao ac)
|
||||||
recent-positions (recent-history (:history ac))
|
recent-positions (recent-history (:history ac))
|
||||||
_ (log-info "WOO %s" (last recent-positions))
|
|
||||||
_ (log-info "%s: Recent history has %s positions, most recent is %s secs old"
|
_ (log-info "%s: Recent history has %s positions, most recent is %s secs old"
|
||||||
icao
|
icao
|
||||||
(count recent-positions)
|
(count recent-positions)
|
||||||
@ -370,7 +382,9 @@
|
|||||||
:layers "venue"
|
:layers "venue"
|
||||||
:size 50})
|
:size 50})
|
||||||
nearby (:features nearby)
|
nearby (:features nearby)
|
||||||
wiki-nearby (filter feature-has-wikipedia-page? nearby)]
|
wiki-nearby (filter feature-has-wikipedia-page? nearby)
|
||||||
|
sqb (if-let [sqb-path (:basestation-sqb config)]
|
||||||
|
(get-basestation-sqb-record icao sqb-path))]
|
||||||
(log-info "%s: Nearby geo search: %s potential landmarks, %s with wikipedia pages"
|
(log-info "%s: Nearby geo search: %s potential landmarks, %s with wikipedia pages"
|
||||||
icao (count nearby) (count wiki-nearby))
|
icao (count nearby) (count wiki-nearby))
|
||||||
(log-info "%s" (->> nearby (take 3) (map :properties)))
|
(log-info "%s" (->> nearby (take 3) (map :properties)))
|
||||||
@ -385,7 +399,7 @@
|
|||||||
icao
|
icao
|
||||||
(get-in f [:properties :label] f)
|
(get-in f [:properties :label] f)
|
||||||
(get-in f [:properties :addendum] f)))
|
(get-in f [:properties :addendum] f)))
|
||||||
(let [description (generate-description ac coarse wiki-nearby nearby)]
|
(let [description (generate-description ac sqb coarse wiki-nearby nearby)]
|
||||||
(log-warn "Description: %s" description)
|
(log-warn "Description: %s" description)
|
||||||
description))])
|
description))])
|
||||||
(fn [[image-path description]]
|
(fn [[image-path description]]
|
||||||
@ -421,13 +435,16 @@
|
|||||||
(.requiredOption "--lon <lat>" "Longitude of the circle of the region of interest" parse-number)
|
(.requiredOption "--lon <lat>" "Longitude of the circle of the region of interest" parse-number)
|
||||||
(.requiredOption "--url <url>" "API url")
|
(.requiredOption "--url <url>" "API url")
|
||||||
(.option "--radius <radius>" "Radius of the circle of interest, in nautical miles" 20 parse-number)
|
(.option "--radius <radius>" "Radius of the circle of interest, in nautical miles" 20 parse-number)
|
||||||
|
(.option "--basestation-sqb <path>" "Path to a basestation.sqb database file")
|
||||||
(.option "--no-tweeting" "Do not tweet.")
|
(.option "--no-tweeting" "Do not tweet.")
|
||||||
(.parse (.-argv js/process)))
|
(.parse (.-argv js/process)))
|
||||||
(let [start-time (current-time)]
|
(let [start-time (current-time)]
|
||||||
(p/then (p/all [(read-history-db history-db-path)
|
(p/then (p/all [(read-history-db history-db-path)
|
||||||
(util/read-config secrets-path)])
|
(util/read-config secrets-path)])
|
||||||
(fn [[db config]]
|
(fn [[db config]]
|
||||||
(p/let [config (assoc config :tweeting-enabled? (.-tweeting commander))
|
(p/let [config (assoc config
|
||||||
|
:tweeting-enabled? (.-tweeting commander)
|
||||||
|
:basestation-sqb (.-basestationSqb commander))
|
||||||
data (get-adsbexchange-live-data
|
data (get-adsbexchange-live-data
|
||||||
{:url (.-url commander)
|
{:url (.-url commander)
|
||||||
:api-key (get-in config [:adsbx :api-key])
|
:api-key (get-in config [:adsbx :api-key])
|
||||||
|
Loading…
Reference in New Issue
Block a user