825 lines
32 KiB
Clojure
825 lines
32 KiB
Clojure
(ns lemondronor.advisorycircular
|
|
(:require
|
|
["commander" :as commander]
|
|
["fs" :as fs]
|
|
[cljs.pprint :as pprint]
|
|
[clojure.set :as set]
|
|
[clojure.string :as string]
|
|
[goog.object]
|
|
[kitchen-async.promise :as p]
|
|
[lemondronor.advisorycircular.adsbx :as adsbx]
|
|
[lemondronor.advisorycircular.generation :as generation]
|
|
[lemondronor.advisorycircular.geo :as geo]
|
|
[lemondronor.advisorycircular.logging :as logging]
|
|
[lemondronor.advisorycircular.pelias :as pelias]
|
|
[lemondronor.advisorycircular.twitter :as twitter]
|
|
[lemondronor.advisorycircular.util :as util]
|
|
["sqlite" :as sqlite]))
|
|
|
|
(declare logger log-debug log-verbose log-info log-warn log-error)
|
|
(logging/deflog "advisorycircular" logger)
|
|
|
|
|
|
(def log-prefix (atom ""))
|
|
|
|
|
|
(defn parse-json [s]
|
|
(.parse js/JSON s))
|
|
|
|
|
|
|
|
(defn create-aircraft-info-db [json-path sqb-path]
|
|
(log-info "Reading %s" json-path)
|
|
(p/let [info-str (util/read-file json-path)
|
|
info (parse-json info-str)
|
|
_ (log-info "Creating DB %s" sqb-path)
|
|
db (sqlite/open sqb-path clj->js {js/Promise js/Promise})]
|
|
(.run db "DROP TABLE IF EXISTS aircraft")
|
|
(.run db "DROP INDEX IF EXISTS idx_aircraft_icao")
|
|
(.run db
|
|
(str "CREATE TABLE aircraft ("
|
|
"icao TEXT NOT NULL PRIMARY KEY,"
|
|
"registration TEXT,"
|
|
"type TEXT"
|
|
");"))
|
|
(.run db "BEGIN TRANSACTION")
|
|
(p/let [count (p/loop [count 0
|
|
keys (seq (goog.object/getKeys info))
|
|
_ nil]
|
|
(when (and (> count 0) (zero? (mod count 100000)))
|
|
(log-info "Inserted %s records" count))
|
|
(if (seq keys)
|
|
(let [icao (first keys)
|
|
rec (aget info icao)
|
|
reg (aget rec "r")
|
|
type (aget rec "d")]
|
|
(p/recur
|
|
(inc count)
|
|
(rest keys)
|
|
(.run db "INSERT INTO aircraft (icao, registration, type) VALUES (?, ?, ?)"
|
|
icao
|
|
(aget rec "r")
|
|
(aget rec "d"))))
|
|
count))]
|
|
(.run db "CREATE UNIQUE INDEX idx_aircraft_icao ON aircraft (icao)")
|
|
(log-info "Committing %s records" count))
|
|
(.run db "COMMIT")))
|
|
|
|
(defn get-aircraft-info-record [icao db-path]
|
|
(log-info "%s: Looking up in %s" icao db-path)
|
|
(p/let [record
|
|
(p/-> (sqlite/open db-path clj->js {js/Promise js/Promise})
|
|
(.get "SELECT registration, type from aircraft where icao = ?" icao))]
|
|
(log-info "%s: aircraft-info record: %s" icao (js->clj record :keywordize-keys true))
|
|
(-> record
|
|
(js->clj :keywordize-keys true))))
|
|
|
|
|
|
(defn parse-adsbexchange-ac-element [e]
|
|
(let [nilstr #(if (= % "") nil %)
|
|
numstr #(if (= % "") nil (js/parseFloat %))
|
|
boolstr #(if (or (= % "1") (= (string/lower-case %) "true"))
|
|
true
|
|
false)]
|
|
{:postime (numstr (e "postime"))
|
|
:lat (numstr (e "lat"))
|
|
:lon (numstr (e "lon"))
|
|
:icao (e "icao")
|
|
:registration (nilstr (e "reg"))
|
|
:alt (numstr (e "alt"))
|
|
:gnd? (boolstr (e "gnd"))
|
|
:mlat? (= (e "mlat") "1")
|
|
:speed (numstr (e "spd"))
|
|
:squawk (nilstr (e "sqk"))
|
|
:military? (= (e "mil") "1")
|
|
:callsign (nilstr (e "call"))
|
|
:type (nilstr (e "type"))}))
|
|
|
|
|
|
(defn parse-adsbexchange-live-data [json-str]
|
|
{:aircraft
|
|
(map parse-adsbexchange-ac-element
|
|
(get
|
|
(js->clj (parse-json json-str))
|
|
"ac"))})
|
|
|
|
|
|
(defn get-adsbexchange-live-data [{:keys [url lat lon radius-nm api-key api-whitelist]}]
|
|
(let [url (->> [url
|
|
"lat" lat
|
|
"lon" lon
|
|
"dist" (.toFixed radius-nm 1)]
|
|
(map str)
|
|
(string/join "/"))
|
|
headers (cond-> {:api-auth api-key}
|
|
api-whitelist (assoc :ADSBX-WL api-whitelist))]
|
|
(p/let [http-result (util/http-get url {:headers headers})]
|
|
(let [result (parse-adsbexchange-live-data http-result)]
|
|
(log-verbose "Got %s aircraft from API" (count (:aircraft result)))
|
|
result))))
|
|
|
|
|
|
;; Given a vector of position history, removes old entries.
|
|
|
|
(defn prune-history [history now config]
|
|
(let [h (filterv #(< (- now (:time %)) (:max-history-age-ms config)) history)]
|
|
h))
|
|
|
|
|
|
(defn update-history-db-record [db ac]
|
|
(let [icao (:icao ac)
|
|
new-history-entry {:lat (:lat ac)
|
|
:lon (:lon ac)
|
|
:alt (:alt ac)
|
|
:gnd? (:gnd? ac)
|
|
:time (:postime ac)}]
|
|
(if (contains? db icao)
|
|
(let [old-record (db icao)
|
|
history (:history (db icao))
|
|
updated-record (-> old-record
|
|
(merge ac)
|
|
(assoc :history (conj history new-history-entry)))]
|
|
(assoc db icao updated-record))
|
|
(assoc db icao (assoc ac :history [new-history-entry])))))
|
|
|
|
|
|
(defn update-history-db-add-new-data [db new-data now]
|
|
(let [initial-count (count db)
|
|
initial-icaos (set (keys db))
|
|
updated-db (reduce update-history-db-record db new-data)
|
|
new-count (count updated-db)
|
|
new-icaos (set/difference (set (keys updated-db)) initial-icaos)]
|
|
(log-verbose "Added %s new aircraft records (%s). %s total."
|
|
(- new-count initial-count)
|
|
(string/join "," new-icaos)
|
|
new-count)
|
|
updated-db))
|
|
|
|
|
|
(defn prune-histories [db now config]
|
|
(reduce-kv (fn [m k v]
|
|
(assoc m k (update v :history prune-history now config)))
|
|
{}
|
|
db))
|
|
|
|
|
|
;; Removes entries for any aircraft that we haven't seen in a while.
|
|
|
|
(defn prune-records [db now]
|
|
(let [initial-count (count db)
|
|
initial-icaos (set (keys db))
|
|
pruned-db (reduce-kv (fn [m k v]
|
|
(if (or (> (count (:history v)) 0)
|
|
(if-let [ended-circling-time (:ended-circling-time v)]
|
|
(< (- now ended-circling-time) (* 30 60 1000))))
|
|
(assoc m k v)
|
|
m))
|
|
{}
|
|
db)
|
|
new-count (count pruned-db)
|
|
pruned-icaos (set/difference initial-icaos (set (keys pruned-db)))]
|
|
(log-verbose "Pruned %s stale aircraft records (%s). %s remain"
|
|
(- initial-count new-count)
|
|
(string/join "," pruned-icaos)
|
|
new-count)
|
|
pruned-db))
|
|
|
|
|
|
(defn debug-print [& args]
|
|
(apply println (drop 1 args))
|
|
(println (first args))
|
|
(first args))
|
|
|
|
|
|
(defn update-history-db [db new-data now config]
|
|
(-> db
|
|
(update-history-db-add-new-data new-data now)
|
|
(prune-histories now config)
|
|
(prune-records now)))
|
|
|
|
|
|
(defn write-history-db [db path]
|
|
(fs/writeFileSync path (.stringify js/JSON (clj->js db) nil " "))
|
|
db)
|
|
|
|
|
|
;; Reads the history database from a path. Returns a promise that
|
|
;; resolves to the database value.
|
|
|
|
(defn read-history-db [path]
|
|
(p/let [json-str (util/read-file path {:encoding "utf-8"})
|
|
db (js->clj (parse-json json-str) :keywordize-keys true)
|
|
db (into {} (map (fn [[k v]] [(name k) v]) db))]
|
|
(log-verbose "Loaded %s aircraft from database %s" (count db) path)
|
|
db))
|
|
|
|
|
|
(defn current-time []
|
|
(/ (.getTime (js/Date.)) 1))
|
|
|
|
|
|
(defn ac-desc [ac]
|
|
(str (:icao ac) " " "http://tar1090.adsbexchange.com/?icao=" (:icao ac)
|
|
" " (:lat ac) " " (:lon ac)
|
|
" " (:registration ac) " " (:alt ac) " " (:curviness ac) " "
|
|
(:normalized-curviness ac)))
|
|
|
|
|
|
(defn map-screenshot
|
|
([icao lat lon now]
|
|
(map-screenshot icao lat lon now {}))
|
|
([icao lat lon now options]
|
|
(p/let [image-path (str (string/join "-" [@log-prefix icao (util/format-utc-ts now)])
|
|
".png")
|
|
_ (adsbx/screenshot-aircraft icao lat lon
|
|
{:timeout 30000
|
|
:output-path image-path
|
|
:headless? true
|
|
:layer (:layer options)
|
|
:zoom (:zoom options)
|
|
;; :viewport {:width 1600 :height 800}
|
|
;; :clip {:width 1600 :height 800 :x 0 :y 0}
|
|
})]
|
|
(log-verbose "%s: Got screenshot" icao)
|
|
image-path)))
|
|
|
|
|
|
(defn airport-data-aircraft-photo [icao reg]
|
|
(log-info "%s: Geting aircraft-data photo" icao)
|
|
(let [query (cond-> {:m icao :n 1}
|
|
reg
|
|
(assoc :r reg))]
|
|
(p/let [r (util/http-get "https://www.airport-data.com/api/ac_thumb.json"
|
|
{:query query})]
|
|
(let [r (js->clj (parse-json r) :keywordize-keys true)]
|
|
(if (and (:data r) (> (count (:data r)) 0))
|
|
(p/let [url (get-in r [:data 0 :image])
|
|
image (util/http-get url {:encoding nil})]
|
|
image)
|
|
(do
|
|
(log-info "%s: No aircraft-data photo available" icao)
|
|
nil))))))
|
|
|
|
|
|
(defn aircraft-photo [icao registration]
|
|
(p/let [photo (p/race [(p/let [_ (util/timeout 7000)]
|
|
:timeout)
|
|
(airport-data-aircraft-photo icao registration)])]
|
|
(if (= photo :timeout)
|
|
(do
|
|
(log-warn "%s: airport-data API timed out" icao)
|
|
nil)
|
|
photo)))
|
|
|
|
|
|
(defn circling? [ac config]
|
|
(and (> (geo/flight-curviness (filter #(not (:gnd? %)) (:history ac)))
|
|
(:curviness-threshold-degrees config))
|
|
(> (:alt ac) 300)))
|
|
|
|
|
|
(defn circling2? [ac config]
|
|
(let [relevant-history (filter #(not (:gnd? %)) (:history ac))]
|
|
(and (> (geo/flight-curviness relevant-history) (:curviness-threshold-degrees config))
|
|
(> (:alt ac) 300))))
|
|
|
|
|
|
;; Returns a vector of two elements,
|
|
;; [updated-database potentially-circling-aircraft]
|
|
|
|
(defn detect-circles [db now config]
|
|
(log-verbose "Detecting circles")
|
|
(loop [old-db (seq db)
|
|
new-db {}
|
|
potential-circles '()]
|
|
(if (seq old-db)
|
|
(let [[icao ac] (first old-db)
|
|
curviness (geo/flight-curviness (:history ac))
|
|
ac (assoc ac
|
|
:curviness curviness
|
|
:normalized-curviness (geo/flight-normalized-curviness (:history ac)))
|
|
currently-circling? (circling2? ac config)
|
|
previously-circling? (:started-circling-time ac)]
|
|
(cond
|
|
(and currently-circling?
|
|
(not previously-circling?)
|
|
(or (nil? (:ended-circling-time ac))
|
|
(> (- now (:ended-circling-time ac)) (* 30 60 1000))))
|
|
(let [new-ac (assoc ac :started-circling-time now)]
|
|
(recur (rest old-db)
|
|
(assoc new-db icao new-ac)
|
|
(conj potential-circles new-ac)))
|
|
(and previously-circling?
|
|
(not currently-circling?))
|
|
(let [started-circling-time (:started-circling-time ac)
|
|
new-ac (assoc ac
|
|
:started-circling-time nil
|
|
:ended-circling-time now)]
|
|
(log-info "%s: Circle terminated after %s secs: %s"
|
|
icao
|
|
(/ (- now started-circling-time) 1000)
|
|
(ac-desc ac))
|
|
(recur (rest old-db)
|
|
(assoc new-db icao new-ac)
|
|
potential-circles))
|
|
:else
|
|
(recur (rest old-db)
|
|
(assoc new-db icao ac)
|
|
potential-circles)))
|
|
[new-db potential-circles])))
|
|
|
|
|
|
(defn parse-number [s]
|
|
(let [v (js/parseFloat s)]
|
|
(if (js/isNaN v)
|
|
(throw (str "Not a number: " s))
|
|
v)))
|
|
|
|
|
|
(defn nearby-airports
|
|
([config lat lon]
|
|
(nearby-airports config lat lon {}))
|
|
([config lat lon options]
|
|
(p/let [radius (or (:radius options) 7)
|
|
results (pelias/nearby (:pelias config)
|
|
lat lon
|
|
{:categories "transport:air:aerodrome"
|
|
:boundary.circle.radius radius})
|
|
blocklist (get-in config [:airport :blocklist] [])
|
|
blocklist-patterns (map #(re-pattern (str "(?i)" %)) blocklist)]
|
|
(->> (:features results)
|
|
(remove (fn [airport] (some #(re-find % (get-in airport [:properties :label]))
|
|
blocklist-patterns)))))))
|
|
|
|
|
|
(defn closest-airport [config lat lon]
|
|
(p/let [airports (nearby-airports config lat lon)]
|
|
(->> airports
|
|
(sort-by #(get-in % [:properties :distance]))
|
|
first)))
|
|
|
|
|
|
(defn airport-geojson [config]
|
|
(p/let [airport->feature (fn [a]
|
|
(let [props (:properties a)]
|
|
{:type "Feature"
|
|
:properties {:shape "Circle"
|
|
:radius (* 1000 (:minimum-airport-distance-km config))
|
|
:name (:label props)}
|
|
:geometry {:type "Point"
|
|
:coordinates (-> a :geometry :coordinates)}}))
|
|
lat (:lat config)
|
|
lon (:lon config)
|
|
airports (nearby-airports config lat lon {:radius (+ (:radius-km config)
|
|
(:minimum-airport-distance-km config))})]
|
|
{:type "FeatureCollection"
|
|
:features (conj
|
|
(map airport->feature airports)
|
|
{:type "Feature"
|
|
:properties {:shape "Circle"
|
|
:radius (* 1000 (:radius-km config))
|
|
:name "Center"}
|
|
:geometry {:type "Point"
|
|
:coordinates [(:lon config) (:lat config)]}})}))
|
|
|
|
|
|
(defn log-table [table keys]
|
|
(let [s (with-out-str (pprint/print-table keys table))
|
|
lines (string/split-lines s)]
|
|
(doseq [l lines]
|
|
(log-info "%s" l))))
|
|
|
|
|
|
(def description-templates
|
|
(map generation/parse-template
|
|
[(str
|
|
"["
|
|
;; BEGIN identity & type group. We choose one of the following:
|
|
;;
|
|
;; 1. Civilian registration w/o type.
|
|
"{registration}|"
|
|
;; 2. Civilian registration w/ type.
|
|
"{registration}, {type|a-an},|"
|
|
;; Military registration w/o type.
|
|
"{militaryregistration}, a military aircraft,|"
|
|
;; Military registration w/ type.
|
|
"{militaryregistration}, a military {type},|"
|
|
;; Civilian w/ unknown registration, and don't have type.
|
|
"Aircraft with unknown registration, ICAO {icao}|"
|
|
;; Civilian w/ unknown registration, and have type.
|
|
"{type} with unknown registration, ICAO {icao}|"
|
|
;; Military with unknown registration.
|
|
"Military aircraft with unknown registration, ICAO {militaryicao}"
|
|
;; END identity & type group.
|
|
"] "
|
|
;; Callsign.
|
|
"?:[(callsign {callsign}) ]"
|
|
;; Ideally neighbourhood as well as city or county, but
|
|
;; maybe just city.
|
|
"is circling over [{neighbourhood}, {locality}|{neighbourhood}, {county}|{locality}|{localadmin}|{name}] "
|
|
;; Altitude.
|
|
"?:[at {alt} feet, ]"
|
|
;; Speed;
|
|
"?:[speed {speed} MPH, ]"
|
|
;; Transponder squawk.
|
|
"?:[squawking {squawk}, ]"
|
|
;; Landmark.
|
|
"?:[{nearbydistance} miles from {nearbylandmark} ]"
|
|
;; Hashtag based on registration.
|
|
"?:[#{registration|hashtag} |#{militaryregistration|hashtag} ]"
|
|
;; tar1090 link.
|
|
"https://tar1090.adsbexchange.com/?icao={icao}&zoom=13")]))
|
|
|
|
|
|
(defn expand-template [data]
|
|
(let [results (take 3 (generation/expand
|
|
description-templates
|
|
data
|
|
{:weights {:militaryregistration 4
|
|
:registration 3
|
|
:militaryicao 2
|
|
:icao 1
|
|
:neighbourhood 3
|
|
:locality 3
|
|
:localadmin 1
|
|
:name 0.5}}))]
|
|
;;(log-info "Top description candidates (%s total):" (count results))
|
|
;;(log-table results [:score :text])
|
|
(first results)))
|
|
|
|
|
|
(defn km->miles [km]
|
|
(* km 0.621371))
|
|
|
|
(defn to-fixed [n d]
|
|
(.toFixed n d))
|
|
|
|
|
|
(defn merge-adsbx-aircraft-db-rec [ac ac-db-rec]
|
|
(cond-> ac
|
|
(and (nil? (:registration ac)) (:registration ac-db-rec))
|
|
(assoc :registration (:registration ac-db-rec))
|
|
true
|
|
(assoc :type (:type ac-db-rec))))
|
|
|
|
|
|
;; Creates a template expansion map from the following:
|
|
;;
|
|
;; * ac - The ADSBX API entry
|
|
;; * ac-db-rec - The aircraft DB record
|
|
;; * reverse - reverse geocoder record
|
|
;; * wiki-nearby - nearby landmarks w/ Wikipedia pages
|
|
;; * nearby - nearby landmarks.
|
|
|
|
(defn template-data [ac ac-db-rec reverse nearby]
|
|
(let [rev-props (:properties reverse)
|
|
nearby (:properties nearby)
|
|
info (cond-> (-> ac
|
|
(dissoc :history :type)
|
|
(merge rev-props)
|
|
(merge-adsbx-aircraft-db-rec ac-db-rec))
|
|
(:military? ac)
|
|
(-> (assoc :militaryregistration (:registration ac)
|
|
:militaryicao (:icao ac)))
|
|
nearby
|
|
(assoc :nearbylandmark (:name nearby)
|
|
:nearbydistance (-> nearby :distance km->miles (to-fixed 2)))
|
|
(:speed ac)
|
|
(assoc :speed (.toFixed (* (:speed ac) 1.15078) 0))
|
|
(= (:registration ac) (:callsign ac))
|
|
(dissoc :callsign)
|
|
;; TODO: If layer is "county", find the nearest city.
|
|
)]
|
|
(log-info "Template data: %s %s" nearby info)
|
|
info))
|
|
|
|
|
|
(defn generate-description [ac ac-db-rec reverse nearby]
|
|
(let [info (template-data ac ac-db-rec reverse nearby)
|
|
expansion (expand-template info)]
|
|
(when (not expansion)
|
|
(log-warn "Info: %s" info))
|
|
(:text expansion)))
|
|
|
|
|
|
(defn feature-has-wikipedia-page? [f]
|
|
(get-in f [:addendum :osm :wikipedia]))
|
|
|
|
|
|
(defn recent-history [history]
|
|
(let [most-recent-time (:time (last history))
|
|
cutoff-time (- most-recent-time (* 6 60 1000))
|
|
recent-hist (filter #(> (:time %) cutoff-time) history)]
|
|
recent-hist))
|
|
|
|
|
|
;; Returns [older, recent].
|
|
|
|
(defn split-history [history]
|
|
(let [most-recent-time (:time (last history))
|
|
cutoff-time (- most-recent-time (* 6 60 1000))]
|
|
(split-with #(< (:time %) cutoff-time) history)))
|
|
|
|
|
|
(defn filter-landmarks [config landmarks]
|
|
(let [block-regexes (map re-pattern (:blocklist config))
|
|
blocked? (fn [l]
|
|
(some #(re-find % (-> l :properties :name)) block-regexes))]
|
|
(filter #(not (blocked? %)) landmarks)))
|
|
|
|
|
|
(defn landmark [config lat lon]
|
|
(p/let [landmarks (p/-> (pelias/nearby
|
|
(:pelias config)
|
|
lat
|
|
lon
|
|
{:boundary.circle.radius 100
|
|
:layers "venue"
|
|
:size 50})
|
|
:features)
|
|
_ (log-info "Nearest landmarks:")
|
|
_ (log-table (->> landmarks (take 3) (map :properties))
|
|
[:distance :label :locality :neighborhood :county :gid])
|
|
filtered-landmarks (filter-landmarks (:landmarks config) landmarks)
|
|
_ (when (not (= (take 3 landmarks) (take 3 filtered-landmarks)))
|
|
(log-info "After filtering landmarks:")
|
|
(log-table (->> landmarks (take 3) (map :properties))
|
|
[:distance :label :locality :neighborhood :county :gid]))]
|
|
(first filtered-landmarks)))
|
|
|
|
|
|
(defn geojson-linestring [coords props]
|
|
{:type "Feature"
|
|
:properties props
|
|
:geometry
|
|
{:type "LineString"
|
|
:coordinates (map (fn [pos]
|
|
[(:lon pos) (:lat pos) (util/feet-to-meters (:alt pos))])
|
|
coords)}})
|
|
|
|
|
|
(defn track->geojson [older-positions recent-positions icao centroid]
|
|
{:type "FeatureCollection"
|
|
:features
|
|
[(geojson-linestring older-positions
|
|
{:stroke "#c0070b"
|
|
:stroke-width 2
|
|
:stroke-opacity 1})
|
|
(geojson-linestring recent-positions
|
|
{:stroke "#f50000"
|
|
:stroke-width 2
|
|
:stroke-opacity 1})
|
|
{:type "Feature"
|
|
:properties {:marker-color "#7e7e7e"
|
|
:marker-size "medium"
|
|
:marker-symbol ""
|
|
:ICAO icao}
|
|
:geometry {:type "Point"
|
|
:coordinates [(:lon centroid) (:lat centroid)]}}]})
|
|
|
|
|
|
(defn process-potential-circle [ac config now]
|
|
(p/let [icao (:icao ac)
|
|
[older-positions recent-positions] (split-history (:history ac))
|
|
_ (log-info "%s: Recent history has %s positions, most recent is %s secs old"
|
|
icao
|
|
(count recent-positions)
|
|
(/ (- now (:time (last recent-positions))) 1000))
|
|
centroid (geo/centroid recent-positions)
|
|
lat (:lat centroid)
|
|
lon (:lon centroid)
|
|
_ (log-info "%s: Recent centroid: %s %s" icao lat lon)
|
|
airport (closest-airport config lat lon)
|
|
airport-properties (:properties airport)]
|
|
(if airport
|
|
(log-info "%s: Closest airport is %s, distance: %s km"
|
|
(:icao ac) (:label airport-properties) (:distance airport-properties))
|
|
(log-info "%s: No airports nearby" (:icao ac)))
|
|
(if (and airport-properties (<= (:distance airport-properties) (:minimum-airport-distance-km config)))
|
|
(log-info "%s: Filtering out because it's %s km (minimum is %s) from %s"
|
|
(:icao ac)
|
|
(:distance airport-properties)
|
|
(:minimum-airport-distance-km config)
|
|
(:label airport-properties)
|
|
())
|
|
;; (let [alts (map :alt recent-positions)
|
|
;; min-alt (apply min alts)
|
|
;; max-alt (apply max alts)
|
|
;; ratio (/ max-alt min-alt)]
|
|
;; (log-info "%s: ratio of min alt to max alt: %s [%s - %s]" icao (.toFixed (/ max-alt min-alt) 1) min-alt max-alt))
|
|
(do
|
|
(p/let [coarse (pelias/reverse (:pelias config) lat lon {:layers "coarse"})]
|
|
(let [coarse (first (:features coarse))]
|
|
(if (:properties coarse)
|
|
(log-info "%s: Reverse geocode: %s" icao (:properties coarse))
|
|
(log-error "%s: Reverse geocode failed: %s" icao coarse))
|
|
;; Note that if we're over the ocean we get null :(
|
|
(p/then (p/all [(map-screenshot (:icao ac) lat lon now (:screenshot config))
|
|
(aircraft-photo (:icao ac) (:registration ac))
|
|
(p/let [nearby (landmark config lat lon)
|
|
;;_ (log-info "WOOO %s" nearby)
|
|
ac-db-rec (if-let [ac-info-db-path (:aircraft-info-db-path config)]
|
|
(get-aircraft-info-record icao ac-info-db-path))]
|
|
(let [description (generate-description ac ac-db-rec coarse nearby)]
|
|
(log-info "%s Description: %s" (:icao ac) description)
|
|
description))])
|
|
(fn [[screenshot-path ac-photo description]]
|
|
(if (or (nil? coarse)
|
|
;; TODO: Filter using the layer hierarchy; we want
|
|
;; anything smaller than "region" (state).
|
|
;;(= (get-in coarse [:properties :name]) "California")
|
|
)
|
|
(log-info "%s: Filtering out because we have insuffucient reverse geo info" (:icao ac))
|
|
(if (and screenshot-path description)
|
|
(p/let [screenshot-image (util/read-file screenshot-path)]
|
|
(if (get-in config [:twitter :enabled?])
|
|
(twitter/tweet (twitter/twit (:twitter config))
|
|
description
|
|
(remove nil? [screenshot-image ac-photo])
|
|
lat
|
|
lon)
|
|
(log-warn "Skipping tweeting"))
|
|
(let [path (str (string/join "-" [@log-prefix icao (util/format-utc-ts now)])
|
|
".geojson")]
|
|
(util/write-file
|
|
path
|
|
(.stringify
|
|
js/JSON
|
|
(clj->js (track->geojson older-positions recent-positions icao centroid)))
|
|
{})))
|
|
(log-warn "Skipping tweet %s %s" screenshot-path description)))))))))))
|
|
|
|
|
|
(defn process-potential-circles [acs config now]
|
|
(p/loop [acs acs]
|
|
(when (seq acs)
|
|
(p/do
|
|
(process-potential-circle (first acs) config now)
|
|
(p/recur (rest acs))))))
|
|
|
|
|
|
(def default-config
|
|
{
|
|
;; We keep position reports going back this far.
|
|
:max-history-age-ms (* 25 60 1000)
|
|
;; This is how many degrees of turning we need to see over
|
|
;; max-history-age-ms ms to consider it a potential circling
|
|
;; aircraft.
|
|
:curviness-threshold-degrees (* 4 360)
|
|
;; If the centroid of the aircraft's positions is less than this
|
|
;; close to an airport, then it's probably just doing flight
|
|
;; training.
|
|
:minimum-airport-distance-km 2.5
|
|
:history-db-path "advisorycircular.json"
|
|
:aircraft-info-db-path "aircraft-info.sqb"
|
|
:twitter {:enabled? true}})
|
|
|
|
|
|
(defn build-config-from-commander [commander]
|
|
(cond-> {}
|
|
(.-adsbxUrl commander)
|
|
(assoc-in [:adsbx :url] (.-adsbxUrl commander))
|
|
;; Note that we're distinguishing from the situation where
|
|
;; --tweeting or --no-tweeting is supplied, in which case the
|
|
;; tweeting field will be true or false, from the situation where
|
|
;; it's not suppled, in which case it will be undefined/nil.
|
|
(not (nil? (.-tweeting commander)))
|
|
(assoc-in [:twitter :enabled?] (.-tweeting commander))
|
|
(.-peliasUrl commander)
|
|
(assoc-in [:pelias :url] (.-peliasUrl commander))
|
|
(.-aircraftInfoDb commander)
|
|
(assoc :aircraft-info-db-path (.-aircraftInfoDb commander))
|
|
(.-history commander)
|
|
(assoc :history-db-path (.-history commander))
|
|
(.-lat commander)
|
|
(assoc :lat (.-lat commander))
|
|
(.-lon commander)
|
|
(assoc :lon (.-lon commander))
|
|
(.-radius commander)
|
|
(assoc :radius-km (.-radius commander))))
|
|
|
|
|
|
(defn build-config [config cli-config secrets]
|
|
(util/deep-merge default-config config cli-config secrets))
|
|
|
|
|
|
(defn validate-config [config]
|
|
(let [required [[:lat]
|
|
[:lon]
|
|
[:radius-km]
|
|
[:aircraft-info-db-path]
|
|
[:adsbx :url]
|
|
[:adsbx :api-key]
|
|
[:pelias :url]]
|
|
present (util/nested-keys config)
|
|
missing1 (set/difference (set required) (set present))
|
|
missing2 (when (get-in config [:twitter :enabled?])
|
|
(let [required (map (fn [key] [:twitter key])
|
|
[:consumer-key :consumer-secret
|
|
:access-token :access-token-secret])]
|
|
(set/difference (set required) (set present))))
|
|
missing (concat missing1 missing2)]
|
|
(when (seq missing)
|
|
(throw (js/Error. (str "Missing configuration values: "
|
|
(string/join ", " (sort-by str missing))))))))
|
|
|
|
|
|
(def default-config-path "config.yaml")
|
|
|
|
(defn main [& args]
|
|
(-> commander
|
|
(.option "--lat <lat>" "Latitude of the circle of region of interest" parse-number)
|
|
(.option "--lon <lat>" "Longitude of the circle of the region of interest" parse-number)
|
|
(.option "--adsbx-url <url>" "ADSBX API url")
|
|
(.option "--pelias-url <url>" "Base pelias geocoder URL")
|
|
(.option "--radius <radius>" "Radius of the circle of interest, in km" parse-number)
|
|
(.option "--aircraft-info-db <path>" "Path to an aircraft info DB file")
|
|
(.option "--tweeting" "Enables tweeting")
|
|
(.option "--no-tweeting" "Do not tweet")
|
|
(.option "--config <path>" "Path to the configuration yaml file")
|
|
(.option "--secrets <path>" "Path to the secrets yaml file" "secrets.yaml")
|
|
(.option "--history <path>" "Path to history/state file" "advisorycircular.json")
|
|
(.option "--log-prefix <prefix>" "Log prefix to use")
|
|
(.option "--airport-geojson" "Generate airport GEOJSON and exit")
|
|
(.option "--create-aircraft-info-db-from-json <json path>" "Generate aircraft info DB and exit")
|
|
|
|
(.option "--test")
|
|
(.option "--icao <icao>")
|
|
(.option "--reg <reg>")
|
|
(.parse (.-argv js/process)))
|
|
(logging/set-log-prefix! (or (.-logPrefix commander) ""))
|
|
(reset! log-prefix (or (.-logPrefix commander) ""))
|
|
(p/try
|
|
(cond
|
|
(.-createAircraftInfoDbFromJson commander)
|
|
(create-aircraft-info-db (.-createAircraftInfoDbFromJson commander) (.-aircraftInfoDb commander))
|
|
(.-airportGeojson commander)
|
|
(p/let [base-config (if-let [config-path (.-config commander)]
|
|
(util/read-config config-path)
|
|
(if (fs/existsSync default-config-path)
|
|
(util/read-config default-config-path)
|
|
{}))
|
|
cli-config (build-config-from-commander commander)
|
|
config (build-config base-config cli-config {})
|
|
geojson (airport-geojson config)]
|
|
(println (.stringify js/JSON (clj->js geojson) nil " ")))
|
|
(.-test commander)
|
|
(p/let [photo (airport-data-aircraft-photo (.-icao commander) (.-reg commander))]
|
|
(fs/writeFileSync "photo.jpg" photo))
|
|
:else
|
|
(let [start-time (current-time)]
|
|
;; If --config-path is specified, definitely try to read that
|
|
;; file. Otherwise, only read config.yaml if it exists.
|
|
(p/let [base-config (if-let [config-path (.-config commander)]
|
|
(util/read-config config-path)
|
|
(if (fs/existsSync default-config-path)
|
|
(util/read-config default-config-path)
|
|
{}))
|
|
cli-config (build-config-from-commander commander)
|
|
secrets (util/read-config (.-secrets commander))
|
|
config (build-config base-config cli-config secrets)
|
|
_ (validate-config config)
|
|
history-db-path (:history-db-path config)
|
|
_ (when (not (fs/existsSync history-db-path))
|
|
(log-info "%s does not exist; creating empty one." history-db-path)
|
|
(write-history-db {} history-db-path))
|
|
db (read-history-db history-db-path)
|
|
data (get-adsbexchange-live-data
|
|
(merge (:adsbx config)
|
|
{:lat (:lat config)
|
|
:lon (:lon config)
|
|
:radius-nm (* (:radius-km config) 0.539957)}))
|
|
now (current-time)
|
|
[new-db potential-circles] (-> db
|
|
(update-history-db (:aircraft data) now config)
|
|
(detect-circles now config))]
|
|
(p/do
|
|
(when potential-circles
|
|
(doseq [ac potential-circles]
|
|
(log-warn "%s: New circle detected: %s" (:icao ac) (ac-desc ac)))
|
|
(process-potential-circles potential-circles config now))
|
|
(write-history-db new-db (:history-db-path config))
|
|
(let [end-time (current-time)]
|
|
(log-info
|
|
"Finished in %f s: %s aircraft; %s circles; top curvinesses: %s"
|
|
(/ (- end-time start-time) 1000)
|
|
(count new-db)
|
|
(count potential-circles)
|
|
(->> (vals new-db)
|
|
(sort-by :curviness)
|
|
reverse
|
|
(take 3)
|
|
(map #(str (:icao %) ":" (.toFixed (:curviness %) 0)))
|
|
(string/join " "))))))))
|
|
(p/catch :default e
|
|
(log-error "%s" e)
|
|
(log-error "%s" (.-stack e))
|
|
(.exit js/process 1))))
|
|
|
|
|
|
;; (.on js/process "unhandledRejection"
|
|
;; (fn [reason promise]
|
|
;; (log-error "Error: %s" (.-message reason))
|
|
;; (println (.-stack reason))
|
|
;; (.exit js/process 1)))
|