Initial commit.

This commit is contained in:
John Wiseman 2019-12-13 23:50:19 -08:00
parent 92494178f1
commit e36e176f8e
16 changed files with 2826 additions and 0 deletions

4
TODO.md Normal file
View File

@ -0,0 +1,4 @@
- [ ] If we don't have neighbourhood or locality (just county), try to use a locality of a nearby venue.
- [ ] Add list with custom airport coords and radii.
- [ ] Lookup registration info. Possible sources: FAA, adsbexchange.com.
- [ ] Use Wikipedia graph to rank landmarks?

1614
package-lock.json generated Normal file

File diff suppressed because it is too large Load Diff

18
package.json Normal file
View File

@ -0,0 +1,18 @@
{
"name": "circlebot",
"version": "0.0.1",
"private": true,
"devDependencies": {
"shadow-cljs": "^2.8.78"
},
"dependencies": {
"commander": "^4.0.1",
"geolib": "^3.2.0",
"js-yaml": "^3.13.1",
"puppeteer": "^2.0.0",
"request": "^2.88.0",
"request-promise-native": "^1.0.8",
"twit": "^2.2.11",
"winston": "^3.2.1"
}
}

20
shadow-cljs.edn Normal file
View File

@ -0,0 +1,20 @@
{:source-paths
["src/dev"
"src/main"
"src/test"]
:dependencies
[[org.clojure/math.combinatorics "0.1.5"]
[com.cemerick/url "0.1.1"]
[fipp "0.6.22"]
[instaparse "1.4.10"]
[kitchen-async "0.1.0-SNAPSHOT"]]
:builds
{:script
{:target :node-script
:main lemondronor.circlebot/main
:output-to "out/script.js"}
:test
{:target :node-test
:output-to "out/node-tests.js"
;;:ns-regexp "-spec$"
:autorun true}}}

View File

@ -0,0 +1,426 @@
(ns lemondronor.circlebot
(:require
["commander" :as commander]
["fs" :as fs]
[cljs.pprint :as pprint]
[cljs.reader :as reader]
[clojure.set :as set]
[clojure.string :as string]
[fipp.edn :as fippedn]
[kitchen-async.promise :as p]
[lemondronor.circlebot.adsbx :as adsbx]
[lemondronor.circlebot.generation :as generation]
[lemondronor.circlebot.geo :as geo]
[lemondronor.circlebot.logging :as logging]
[lemondronor.circlebot.pelias :as pelias]
[lemondronor.circlebot.twitter :as twitter]
[lemondronor.circlebot.util :as util]))
(logging/deflog "circlebot" logger)
(defn parse-adsbexchange-ac-element [e]
(let [nilstr #(if (= % "") nil %)
numstr #(if (= % "") nil (js/parseFloat %))]
{:postime (numstr (e "postime"))
:lat (numstr (e "lat"))
:lon (numstr (e "lon"))
:icao (e "icao")
:registration (e "reg")
:alt (numstr (e "alt"))
:mlat? (= (e "mlat") "1")
:speed (numstr (e "spd"))
:squawk (nilstr (e "sqk"))
:military? (= (e "mil") "1")
:callsign (nilstr (e "call"))
:type (nilstr (e "type"))}))
(defn parse-adsbexchange-live-data [json-str]
{:aircraft
(map parse-adsbexchange-ac-element
(get
(js->clj (.parse js/JSON json-str))
"ac"))})
(defn get-adsbexchange-live-data [{:keys [url lat lon radius-nm api-key]}]
(let [url (->> [url
"lat" lat
"lon" lon
"dist" radius-nm]
(map str)
(string/join "/"))]
(p/let [http-result (util/http-get url {:headers {:api-auth api-key}})]
(let [result (parse-adsbexchange-live-data http-result)]
(log-verbose "Got %s aircraft from API" (count (:aircraft result)))
result))))
;; We keep position reports going back this far.
(def max-history-age-ms (* 25 60 1000))
;; Given a vector of position history, removes old entries.
(defn prune-history [history now]
(let [h (filterv #(< (- now (:time %)) max-history-age-ms) history)]
h))
(defn update-history-db-record [db ac]
(let [icao (:icao ac)
new-history-entry {:lat (:lat ac)
:lon (:lon ac)
:time (:postime ac)}]
(if (contains? db icao)
(let [old-record (db icao)
history (:history (db icao))
updated-record (-> old-record
(merge ac)
(assoc :history (conj history new-history-entry)))]
(assoc db icao updated-record))
(assoc db icao (assoc ac :history [new-history-entry])))))
(defn update-history-db-add-new-data [db new-data now]
(let [initial-count (count db)
initial-icaos (set (keys db))
updated-db (reduce update-history-db-record db new-data)
new-count (count updated-db)
new-icaos (set/difference (set (keys updated-db)) initial-icaos)]
(log-verbose "Added %s new aircraft records (%s). %s total."
(- new-count initial-count)
(string/join "," new-icaos)
new-count)
updated-db))
(defn prune-histories [db now]
(reduce-kv (fn [m k v]
(assoc m k (update v :history prune-history now)))
{}
db))
;; Removes entries for any aircraft that we haven't seen in a while.
(defn prune-records [db now]
(let [initial-count (count db)
initial-icaos (set (keys db))
pruned-db (reduce-kv (fn [m k v]
(if (or (> (count (:history v)) 0)
(if-let [ended-circling-time (:ended-circling-time v)]
(< (- now ended-circling-time) (* 20 60 1000))))
(assoc m k v)
m))
{}
db)
new-count (count pruned-db)
pruned-icaos (set/difference initial-icaos (set (keys pruned-db)))]
(log-verbose "Pruned %s stale aircraft records (%s). %s remain"
(- initial-count new-count)
(string/join "," pruned-icaos)
new-count)
pruned-db))
(defn debug-print [& args]
(apply println (drop 1 args))
(println (first args))
(first args))
(defn update-history-db [db new-data now]
(-> db
(update-history-db-add-new-data new-data now)
(prune-histories now)
(prune-records now)))
(defn write-history-db [db path]
(fs/writeFileSync path (with-out-str (fippedn/pprint db)))
db)
;; Reads the history database from a path. Returns a promise that
;; resolves to the database value.
(defn read-history-db [path]
(p/let [edn-str (util/read-file path {:encoding "utf-8"})
db (reader/read-string edn-str)]
(log-verbose "Loaded %s aircraft from database %s" (count db) path)
db))
(defn current-time []
(/ (.getTime (js/Date.)) 1))
;; This is how many degrees of turning we need to see over
;; max-history-age-ms ms to consider it a potential circling aircraft.
(def curviness-threshold-degrees 1440)
(defn ac-desc [ac]
(str (:icao ac) " " (:lat ac) " " (:lon ac)
" #" (:registration ac) " " (:alt ac) " " (:curviness ac) " "
(:normalized-curviness ac)))
(defn screenshot [icao lat lon]
(p/let [image-path
(adsbx/screenshot-aircraft icao lat lon
{:timeout 30000
;;:headless? false
;; :viewport {:width 1600 :height 800}
;; :clip {:width 1600 :height 800 :x 0 :y 0}
:vrs-settings (fs/readFileSync "vrs-settings.json" "utf-8")})]
(log-warn "%s: Got screenshot" icao)
image-path))
(defn circling? [ac]
(and (> (geo/flight-curviness (:history ac)) curviness-threshold-degrees)
(> (:alt ac) 300)))
;; Returns a vector of two elements,
;; [updated-database potentially-circling-aircraft]
(defn detect-circles [db now]
(log-verbose "Detecting circles")
(loop [old-db (seq db)
new-db {}
potential-circles '()]
(if (seq old-db)
(let [[icao ac] (first old-db)
curviness (geo/flight-curviness (:history ac))
ac (assoc ac
:curviness curviness
:normalized-curviness (geo/flight-normalized-curviness (:history ac)))
currently-circling? (circling? ac)
previously-circling? (:started-circling-time ac)]
(cond
(and currently-circling?
(not previously-circling?)
(or (nil? (:ended-circling-time ac))
(> (- now (:ended-circling-time ac)) (* 20 60 1000))))
(let [new-ac (assoc ac :started-circling-time now)]
(recur (rest old-db)
(assoc new-db icao new-ac)
(conj potential-circles new-ac)))
(and previously-circling?
(not currently-circling?))
(let [started-circling-time (:started-circling-time ac)
new-ac (assoc ac
:started-circling-time nil
:ended-circling-time now)]
(log-info "%s: Circle terminated after %s secs: %s"
icao
(/ (- now started-circling-time) 1000)
(ac-desc ac))
(recur (rest old-db)
(assoc new-db icao new-ac)
potential-circles))
:else
(recur (rest old-db)
(assoc new-db icao ac)
potential-circles)))
[new-db potential-circles])))
(defn parse-number [s]
(let [v (js/parseFloat s)]
(if (js/isNaN v)
(throw (str "Not a number: " s))
v)))
(defn debug-prn [x msg]
(println msg (with-out-str (fippedn/pprint x)))
x)
(defn closest-airport [lat lon]
(p/let [results (pelias/nearby lat lon
{:categories "transport:air:aerodrome"
:boundary.circle.radius 7})]
(-> results
(get :features)
(->> (sort-by #(get-in % [:properties :distance])))
first)))
(defn log-table [table keys]
(let [s (with-out-str (pprint/print-table keys table))
lines (string/split-lines s)]
(doseq [l lines]
(log-info "%s" l)))
)
(def description-templates
(map generation/parse-template
[(str "[{registration}|{militaryregistration}, a military aircraft,|"
"Aircraft with unknown registration, ICAO {icao}|"
"Military aircraft with unknown registration, ICAO {militaryicao}] "
"?:[(callsign {callsign}) ]"
"is circling over [{neighbourhood}, {locality}|{neighbourhood}, {county}|{locality}] "
"?:[at {alt} feet, ]"
"?:[speed {speed} MPH, ]"
"?:[squawking {squawk}, ]"
"?:[{nearbydistance} miles from {nearbylandmark} ]"
"?:[#{registration}|#{militaryregistration}]")]))
(defn expand-template [data]
(let [results (take 3 (generation/expand
description-templates
data
{:weights {:militaryregistration 4
:registration 3
:militaryicao 2
:icao 1
:neighbourhood 3
:locality 3}}))]
(log-table results [:score :text])
(first results)))
(defn generate-description [ac reverse wiki-nearby nearby]
(let [rev-props (:properties reverse)
nearby (:properties (first nearby))
wiki-nearby (:properties (first wiki-nearby))
info (cond-> (-> ac (dissoc :history) (merge rev-props))
(:military? ac)
(-> (assoc :militaryregistration (:registration ac)
:militaryicao (:icao ac)))
wiki-nearby
(assoc :nearbylandmark (:name wiki-nearby)
:nearbydistance (:distance wiki-nearby))
(and nearby (not wiki-nearby))
(assoc :nearbylandmark (:name nearby)
:nearbydistance (:distance nearby))
(:speed ac)
(assoc :speed (.toFixed (* (:speed ac) 1.15078) 0))
(= (:registration ac) (:callsign ac))
(dissoc :callsign)
;; TODO: If layer is "county", find the nearest city.
)
expansion (expand-template info)]
(log-info "Description data: %s" info)
(log-info "Description [score: %s] %s" (:score expansion) (:text expansion))
(:text expansion)))
(defn feature-has-wikipedia-page? [f]
(get-in f [:addendum :osm :wikipedia]))
;; If the centroid of the aircraft's positions is less than this close
;; to an airport, then it's probably just doinf flight training.
;;(def minimum-airport-distance-miles 2.5)
(def minimum-airport-distance-miles 0)
(defn process-potential-circle [ac config now]
(p/let [icao (:icao ac)
centroid (geo/centroid (filter #(< (- now (:time %)) (* 3 60 1000)) (:history ac)))
lat (:lat centroid)
lon (:lon centroid)
airport (closest-airport lat lon)
airport-properties (:properties airport)]
(log-info "%s: Recent centroid is %s %s" icao lat lon)
(if airport
(log-info "%s: Closest airport is %s, distance: %s"
(:icao ac) (:label airport-properties) (:distance airport-properties))
(log-info "%s: No airports nearby" (:icao ac)))
(if (and airport-properties (<= (:distance airport-properties) minimum-airport-distance-miles))
(log-info "%s: Filtering out because it's %s miles (minimum is %s) from %s"
(:icao ac)
(:distance airport-properties)
minimum-airport-distance-miles
(:label airport-properties)
())
(do
(p/let [coarse (pelias/reverse lat lon {:layers "coarse"})]
(let [coarse (first (:features coarse))]
(log-info "%s: Reverse geocode: %s" icao (:properties coarse))
;; Note that if we're over the ocean we get null :(
(if (or (nil? coarse)
;; TODO: Filter using the layer hierarchy; we want
;; anything smaller than "region" (state).
(= (get-in coarse [:properties :name]) "California"))
(log-info "%s: Filtering out because it is outside Los Angeles County" (:icao ac))
(p/then (p/all [(screenshot (:icao ac) lat lon)
(p/let [nearby (pelias/nearby lat lon {:boundary.circle.radius 100
:layers "venue"
:size 50})
nearby (:features nearby)
wiki-nearby (filter feature-has-wikipedia-page? nearby)]
(log-info "%s: Nearby geo search: %s potential landmarks, %s with wikipedia pages"
icao (count nearby) (count wiki-nearby))
(doseq [f wiki-nearby]
(log-info "%s: %s %s"
icao
(get-in f [:properties :label] f)
(get-in f [:properties :addendum] f)))
(let [description (generate-description ac coarse wiki-nearby nearby)]
(log-warn "Description: %s" description)
description))])
(fn [[image-path description]]
(if (and image-path description)
(if (:twitter config)
(twitter/tweet (twitter/twit (:twitter config))
description
[image-path])
(log-warn "Skipping tweeting: No twitter config provided"))
(log-warn "Skipping tweet %s %s" image-path description)))))))))))
(defn process-potential-circles [acs config now]
(p/loop [acs acs]
(when (seq acs)
(p/do
(process-potential-circle (first acs) config now)
(p/recur (rest acs))))))
(def history-db-path "advisory-circular.db")
(def secrets-path "secrets.yaml")
(defn main [& args]
(-> commander
(.requiredOption "--lat <lat>" "Latitude of the circle of region of interest" parse-number)
(.requiredOption "--lon <lat>" "Longitude of the circle of the region of interest" parse-number)
(.requiredOption "--url <url>" "API url")
(.option "--queue <queue url>" "Queue URL (graphile)")
(.option "--radius <radius>" "Radius of the circle of interest, in nautical miles" 20 parse-number)
(.parse (.-argv js/process)))
(let [start-time (current-time)]
(p/then (p/all [(read-history-db history-db-path)
(util/read-config secrets-path)])
(fn [[db config]]
(p/let [data (get-adsbexchange-live-data
{:url (.-url commander)
:api-key (get-in config [:adsbx :api-key])
:lat (.-lat commander)
:lon (.-lon commander)
:radius-nm (.-radius commander)})
now (current-time)
[new-db potential-circles] (-> db
(update-history-db (:aircraft data) now)
(detect-circles now))]
(p/do
(when potential-circles
(doseq [ac potential-circles]
(log-warn "%s: New circle detected: %s" (:icao ac) (ac-desc ac)))
(process-potential-circles potential-circles config now))
(write-history-db new-db history-db-path)
(let [end-time (current-time)]
(log-info
"Completed processing in %s seconds: tracking %s aircraft; %s potential circles"
(/ (- end-time start-time) 1000)
(count new-db)
(count potential-circles)))))))))

View File

@ -0,0 +1,161 @@
;; Scripts a real live chromium browser to take screenshots from ADS-B
;; Exchange.
(ns lemondronor.circlebot.adsbx
(:require
["fs" :as fs]
["puppeteer" :as puppeteer]
[clojure.string :as string]
[goog.string :as gstring]
[kitchen-async.promise :as p]
[lemondronor.circlebot.logging :as logging]
goog.string.format))
(logging/deflog "adsbx" logger)
(def user-agent "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/53 (KHTML, like Gecko) Chrome/15.0.87")
;; (defn wait-for-property [page prop-string]
;; (let [pieces (string/split prop-string ".")]
;; (p/loop [pieces-checking (take 1 pieces)
;; pieces-remaining (drop 1 pieces)
;; ]
;; (println "Waiting for" (string/join "." pieces-checking))
;; (p/resolve (.waitForFunction page (string/join "." pieces-checking)))
;; (when (seq pieces-remaining)
;; (recur (concat pieces-checking (take 1 pieces-remaining))
;; (drop 1 pieces-remaining))))))
(defn timeout
([ms]
(timeout ms nil))
([ms v]
(p/promise [resolve]
(js/setTimeout #(resolve v) ms))))
(defn current-time-secs []
(/ (.getTime (js/Date.)) 1000))
(defn import-settings [page settings]
(let [import-settings-btn-xpath "//button[contains(text(), 'Import Settings')]"]
(p/try
(log-debug "Importing settings")
(p/all [(.waitForNavigation page)
(.waitForXPath page import-settings-btn-xpath)
(.goto page
"https://global.adsbexchange.com/VirtualRadar/settings.html"
(clj->js {:referer "https://adsbexchange.com/"}))])
(timeout 50)
(p/let [import-settings-btn (.$x page import-settings-btn-xpath)]
(.click (nth import-settings-btn 0))
(.waitForSelector page "textarea")
(p/let [textarea (.$x page "//textarea")]
(.evaluate page
(fn [el value]
(set! (.-value el) value))
(nth textarea 0)
settings))
(p/let [import-btn (.$x page "//button[text()='Import']")]
(.click (nth import-btn 0))
(timeout 100)))
(log-debug "Imported settings"))))
(defn screenshot-aircraft
([icao lat lon]
(screenshot-aircraft icao lat lon {}))
([icao lat lon options]
(log-info "Taking screenshot of aircraft %s at %s %s" icao lat lon)
(let [icao-selector (gstring/format "//td[contains(., '%s')]"
(string/upper-case icao))
launch-options {:headless (get options :headless? true)
:defaultViewport (get options :viewport {:width 1600
:height 800})}
took-screenshot?_ (atom false)]
(p/race
[(p/promise [resolve reject]
(js/setTimeout
#(when-not @took-screenshot?_
(log-error "Screenshot timing out")
(reject (js/Error. "Timeout")))
(get options :timeout 30000)))
(p/let [browser (puppeteer/launch (clj->js launch-options))
page (.newPage browser)]
(if-let [user-agent (:user-agent options)]
(.setUserAgent page user-agent))
(if-let [vrs-settings (:vrs-settings options)]
(import-settings page vrs-settings))
(log-debug "Navigating")
(p/all [(.waitForNavigation page)
(.goto page
"https://global.adsbexchange.com/VirtualRadar/desktop.html"
(clj->js {:referer "https://adsbexchange.com/"}))])
;; Wait until the page has loaded the pan and zoom functions
;; we need.
(log-debug "Waiting for script objects")
(.waitForFunction page "window.VRS")
(.waitForFunction page "VRS.bootstrap")
(.waitForFunction page "VRS.bootstrap.pageSettings")
(.waitForFunction page "VRS.bootstrap.pageSettings.mapPlugin")
(log-debug "Xooming")
;; Zoom.
(.evaluate
page
(gstring/format "VRS.bootstrap.pageSettings.mapPlugin.setZoom(%s);" (get options :zoom 13)))
(timeout 500)
;; Pan to the coordinates of interest.
(log-debug "Panning")
(.evaluate
page
(gstring/format
"VRS.bootstrap.pageSettings.mapPlugin.panTo({lat:%f,lng:%f});"
lat
lon))
;; Once we pan to our area of interest it can take some time
;; for VRS to load in the map tiles and the aircraft present
;; in that area. So we wait before clicking.
(log-debug "Waiting for aircraft")
(.waitForXPath page icao-selector)
;; Now try to click on the aircraft of interest in the list
;; display.
(log-debug "Selecting aircraft")
(p/let [aircraft-entry (.$x page icao-selector)]
(.click (nth aircraft-entry 0)))
;; Click "Show on map" to center it.
(p/let [show-on-map (.$x page "//a[contains(., 'Show on map')]")]
(.click (nth show-on-map 0)))
;; Wait for tiles to load. I wonder if we could try something
;; like this:
;; https://stackoverflow.com/questions/22288933/how-to-tell-when-all-visible-tiles-have-fully-loaded
(timeout 3000)
(let [path (get options :output-path (str "screenshot-"
(.toFixed (current-time-secs) 0)
"-"
icao
".png"))]
(log-info "Writing %s" path)
(p/do
(.screenshot page (clj->js {:path path
:clip (get options :clip {:width 860
:height 680
:x 46
:y 46})}))
(log-info "screenshot done")
(reset! took-screenshot?_ true)
(log-info "closing browser")
(.close browser)
(log-info "closing browser done")
path)))]))))
;; (defn main [& args]
;; (let [settings (fs/readFileSync "vrs-settings.json" "utf-8")
;; zoom 13
;; icao (nth args 0)
;; lat (js/parseFloat (nth args 1))
;; lon (js/parseFloat (nth args 2))]
;; (screenshot-aircraft icao lat lon {:zoom zoom :vrs-settings settings})))

View File

@ -0,0 +1,110 @@
(ns lemondronor.circlebot.generation
(:require
[clojure.math.combinatorics :as combo]
[clojure.string :as string]
[instaparse.core :as insta]))
(def ^:private %parse-template
(insta/parser
"<pattern> = term | implicit-sequence
implicit-sequence = term (term)+
<term> = sequence | optional | choice | varref | text
<no-ws-term> = sequence | optional | choice | varref | no-ws-text
sequence = <'['> term+ <']'>
optional = <'⁇'> no-ws-term
choice = <'['> pattern <'|'> pattern (<'|'> pattern)* <']'>
varref = <'{'> #'[a-zA-Z\\_][a-zA-Z0-9\\-\\_\\.]+' <'}'>
text = #'[^\\{\\[\\]\\|⁇]+'
no-ws-text = #'[^\\{\\[\\]\\|⁇\\s]+'
"))
;; Because I don't know enough to write the correct negative-lookahead
;; regex so that "?:" doesn't get parsed into a `text` or
;; `no-ws-text`, I lex it into a unicode character that I'm sure no
;; one will ever use (right?).
(defn ^:private lex-template [template]
(string/replace template "?:" "⁇"))
(defn parse-template [template]
(let [result (-> template lex-template %parse-template)
;; Convert :implicit-sequence into :sequence and :no-ws-text
;; into :text.
xformed-result (insta/transform
{:implicit-sequence (fn [& children]
(into [:sequence] children))
:no-ws-text (fn [text]
[:text text])}
result)]
;; Put in the implicit :sequence if necessary.
(if (= (count xformed-result) 1)
(first xformed-result)
(into [:sequence] xformed-result))))
(defmulti expand% (fn [template data] (first template)))
(defmethod expand% :varref [template data]
(let [var (keyword (second template))
val (data var)]
(if (or (nil? val) (= val ""))
'()
(list {:varrefs [var] :text (str (data var))}))))
(defmethod expand% :text [template data]
(list {:varrefs [] :text (second template)}))
(defmethod expand% :optional [template data]
(concat (list {:varrefs [] :text ""})
(expand% (second template) data)))
(defmethod expand% :choice [template data]
(apply concat (map #(expand% % data) (rest template))))
(defmethod expand% :sequence [template data]
(let [merge-expansions1 (fn
([a] a)
([a b] {:varrefs (concat (:varrefs a) (:varrefs b))
:text (str (:text a) (:text b))}))
merge-expansions (fn [args]
(reduce merge-expansions1 args))
things (map #(expand% % data) (rest template))
chains (apply combo/cartesian-product things)]
(map merge-expansions chains)))
;; A simple expansion scorer that gives high scores to expansions that
;; use more variables, with optional per-variable weights.
(defn score-by-varref-count [expansion weights]
(assoc expansion
:score (reduce + (map #(weights % 1) (:varrefs expansion)))))
(defn expand
([templates data]
(expand templates data {}))
([templates data options]
(->> (apply concat (map #(expand% % data) templates))
(map (get options :scorer #(score-by-varref-count % (get options :weights {}))))
(sort-by :score)
reverse)))
(defn generate-all
([templates data]
(generate-all templates data {}))
([templates data options]
(->> (expand templates data options)
(map :text))))
(defn generate
([templates data]
(generate templates data {}))
([templates data options]
(-> (generate-all templates data options)
first)))

View File

@ -0,0 +1,112 @@
;; Most of this was written before we started using geolib. Maybe
;; replace with geolib functions?
(ns lemondronor.circlebot.geo
(:require
[lemondronor.circlebot.logging :as logging]
["geolib" :as geolib]))
(logging/deflog "geo" logger)
(def to-radians
#?(:clj Math/toRadians
:cljs (fn [d] (/ (* d Math/PI) 180))))
(def to-degrees
#?(:clj Math/toDegrees
:cljs (fn [d] (/ (* d 180) Math/PI))))
(defn centroid [coords]
(let [center (js->clj (geolib/getCenter (clj->js coords)) :keywordize-keys true)
xcenter {:lat (:latitude center)
:lon (:longitude center)}]
xcenter))
(defn distance
"Returns the distance between two positions, in km."
[pos1 pos2]
(let [earth-radius 6372.8 ;; km
sin2 (fn sin2 ^double [^double theta] (* (Math/sin theta) (Math/sin theta)))
alpha (fn alpha ^double [^double lat1 ^double lat2 ^double delta-lat ^double delta-lon]
(+ (sin2 (/ delta-lat 2.0))
(* (sin2 (/ delta-lon 2)) (Math/cos lat1) (Math/cos lat2))))
{lat1 :lat lon1 :lon} pos1
{lat2 :lat lon2 :lon} pos2
delta-lat (to-radians (- ^double lat2 ^double lat1))
delta-lon (to-radians (- ^double lon2 ^double lon1))
lat1 (to-radians lat1)
lat2 (to-radians lat2)]
(* earth-radius 2
(Math/asin (Math/sqrt (alpha lat1 lat2 delta-lat delta-lon))))))
(defn position= [p1 p2]
(= p1 p2))
(defn bearing
"Returns the bearing from one position to another, in degrees."
[pos1 pos2]
(if (position= pos1 pos2)
nil
(let [{lat1 :lat lon1 :lon} pos1
{lat2 :lat lon2 :lon} pos2
lat1 ^double (to-radians lat1)
lat2 ^double (to-radians lat2)
lon-diff ^double (to-radians (- ^double lon2 ^double lon1))
y ^double (* (Math/sin lon-diff) (Math/cos lat2))
x ^double (- (* (Math/cos lat1) (Math/sin lat2))
(* (Math/sin lat1) (Math/cos lat2) (Math/cos lon-diff)))]
(mod (+ (to-degrees (Math/atan2 y x)) 360.0) 360.0))))
(defn bearing-diff
"Computes difference between two bearings. Result is [-180, 180]."
^double [a b]
(let [d (- 180.0 (mod (+ (- a b) 180.0) 360.0))]
d))
(defn spurious-bearing [bearing]
(nil? bearing))
(defn spurious-bearing-diff [^double bearing]
(> (Math/abs bearing) 160))
(defn curviness
"Computes the total curvature of a sequence of bearings."
^double [bearings]
(Math/abs
^double (reduce (fn [^double sum [^double a ^double b]]
(let [d (bearing-diff a b)]
(if (spurious-bearing-diff d)
sum
(+ sum d))))
0.0
(partition 2 1 bearings))))
;; flight is a vector of {:lat <lat> :lon <lon}.
(defn flight-curviness [flight]
(->> (partition 2 1 flight)
(map #(apply bearing %))
(filter #(not (spurious-bearing %)))
curviness))
(defn flight-distance [flight]
(->> (partition 2 1 flight)
(map #(apply distance %))
(apply +)))
(defn flight-normalized-curviness [flight]
(/ (flight-curviness flight)
(flight-distance flight)))

View File

@ -0,0 +1,11 @@
(ns lemondronor.circlebot.logging)
(defmacro deflog [service name]
`(do
(def ~name (logging/get-logger ~service))
(let [log# (.bind (.-log ~name) ~name)]
(defn ~'log-debug [& args#] (apply log# "debug" args#))
(defn ~'log-verbose [& args#] (apply log# "verbose" args#))
(defn ~'log-warn [& args#] (apply log# "warn" args#))
(defn ~'log-info [& args#] (apply log# "info" args#))
(defn ~'log-error [& args#] (apply log# "error" args#)))))

View File

@ -0,0 +1,37 @@
(ns lemondronor.circlebot.logging
(:require
[goog.string :as gstring]
goog.string.format
["winston" :as winston])
(:require-macros
[lemondronor.circlebot.logging]))
(let [createLogger (.-createLogger winston)
format (.-format winston)
transports (.-transports winston)
printf-fmt #(gstring/format "%s%-7s %-9s/%-18s| %s"
(.-timestamp %)
(.-ms %)
(.-service %)
(.-level %)
(.-message %))]
(def logger (createLogger
#js {:level "verbose"
:format (.combine
format
(.colorize format #js {:all true})
(.timestamp format #js {:format "YYYYMMDD HHmmss"})
(.errors format #js {:stack true})
(.splat format)
(.timestamp format)
(.label format)
(.ms format)
(.json format))
:defaultMeta #js {}}))
(.add logger (new (.-Console transports)
#js {:format (.combine format
(.printf format printf-fmt))})))
(defn get-logger [service]
(.child logger #js {:service service}))

View File

@ -0,0 +1,47 @@
;; Simple interface to the pelias geocoder/reverse geocoder.
(ns lemondronor.circlebot.pelias
(:refer-clojure :exclude [reverse])
(:require
[cemerick.url :as c-url]
[kitchen-async.promise :as p]
[lemondronor.circlebot.logging :as logging]
[lemondronor.circlebot.util :as util]))
(logging/deflog "pelias" logger)
(def base-pelias-url "http://raytheon.local:4000/v1")
;; Does an HTTP GET to a pelias API url. Returns a promise that
;; resolves to the API results.
(defn pelias-get [base-url endpoint options]
(p/let [json-str (util/http-get (c-url/url base-url endpoint) options)]
(let [x (js->clj (.parse js/JSON json-str) :keywordize-keys true)]
x)))
;; Performs a pelias "nearby" query. Returns a promsie that resolves
;; to the query results.
(defn nearby
([lat lon options]
(log-verbose "Performing nearby query %s %s %s" lat lon options)
(pelias-get base-pelias-url "nearby"
{:query (assoc options
:point.lat lat
:point.lon lon)})))
;; Performs a pelias "reverse" query. Retuns a promise that resolves
;; to the query result.
(defn reverse
([lat lon options]
(log-verbose "Performing reverse query %s %s %s" lat lon options)
(pelias-get base-pelias-url "reverse"
{:query (assoc options
:point.lat lat
:point.lon lon)})))

View File

@ -0,0 +1,47 @@
(ns lemondronor.circlebot.twitter
(:require
["fs" :as fs]
[kitchen-async.promise :as p]
[lemondronor.circlebot.logging :as logging]
[lemondronor.circlebot.util :as util]
["twit" :as Twit]))
(def fsprom (.-promises fs))
(logging/deflog "twitter" logger)
;; Creates a new twit object.
(defn twit [config]
(Twit. (clj->js {:consumer_key (:consumer-key config)
:consumer_secret (:consumer-secret config)
:access_token (:access-token config)
:access_token_secret (:access-token-secret config)})))
;; Uploads an image to twitter. Returns a promise that resolves to the
;; new media ID of the image.
(defn upload-image [twit path]
(log-info "Uploading media to twitter: %s" path)
(p/let [b64content (util/read-file path {:encoding "base64"})
result (.post twit "media/upload" (clj->js {:media_data b64content}))
media-id (get-in (js->clj result :keywordize-keys true) [:data :media_id_string])]
(log-info "%s got media ID %s" path media-id)
media-id))
;; Posts a tweet with optional multiple media. Returns a promise that
;; resolves to the response result.
(defn tweet [twit status image-paths]
(p/then (p/all (map #(upload-image twit %) image-paths))
(fn [media-ids]
(log-info "Tweeting status:'%s' with media: %s" status media-ids)
(p/let [result (.post twit "statuses/update"
(clj->js {:status status
:media_ids [media-ids]}))
result (js->clj result :keywordize-keys true)]
(log-info "Tweet posted")
result))))

View File

@ -0,0 +1,50 @@
(ns lemondronor.circlebot.util
(:require [cemerick.url :as c-url]
["fs" :as fs]
[kitchen-async.promise :as p]
[lemondronor.circlebot.logging :as logging]
["request-promise-native" :as request]
["js-yaml" :as yaml]))
(logging/deflog "util" logger)
(def fs-promises (.-promises fs))
;; Reads a file, returns a promise resolving to the file contents.
(defn read-file [path options]
(.readFile fs-promises path (clj->js options)))
;; Writes a file, returns a promise that resolves to no arguments on success.
(defn write-file [path data options]
(.writeFile fs-promises path data (clj->js options)))
;; Reads a YAML config file. Returns a promise that resolves to the parsed YAML.
(defn read-config [path]
(log-verbose "Reading config file %s" path)
(p/let [data (read-file path {:encoding "utf-8"})]
(-> (yaml/safeLoad data)
(js->clj :keywordize-keys true))))
;; Fetches a URL. Returns a promise that resolves to the body of the
;; response.
(defn http-get
([url]
(http-get url {}))
([url options]
(let [query (or (:query options) {})
options (dissoc options :query)
url (-> (c-url/url (str url))
(assoc :query query))]
(p/do
(log-info "Fetching %s" url)
(p/let [result (.get request
(clj->js (merge {:url (str url) :gzip true} options)))]
result)))))

View File

@ -0,0 +1,66 @@
(ns lemondronor.circlebot.generation-test
(:require [cljs.test :refer (deftest is)]
[lemondronor.circlebot.generation :as generation]))
(deftest parse-template
(is (= [:optional [:varref "woo"]]
(generation/parse-template "?:{woo}")))
(is (= [:sequence [:text "Hello "] [:varref "foo"] [:text "!"]]
(generation/parse-template "Hello {foo}!")))
(is (= [:sequence
[:text "Hi "] [:varref "name"] [:text ", "]
[:optional [:varref "woo"]]
[:text " "]
[:choice
[:text "What up?"]
[:text "Seeya"]]]
(generation/parse-template "Hi {name}, ?:{woo} [What up?|Seeya]")))
(is (= [:sequence
[:text "Hi "] [:varref "name"] [:text ", "]
[:optional
[:sequence [:varref "woo"] [:text " "]]]
[:choice [:text "What up?"] [:text "Seeya"]]]
(generation/parse-template "Hi {name}, ?:[{woo} ][What up?|Seeya]"))))
(deftest generate
(is (= ["Hello!"]
(generation/generate-all
[(generation/parse-template "Hello!")]
{})))
(is (= "Hello!"
(generation/generate
[(generation/parse-template "Hello!")]
{}))))
(deftest generate-test
(is (= [{:varrefs [], :text ""} {:varrefs [], :text "woo"}]
(generation/expand%
(generation/parse-template "?:woo"))))
(is (= [{:varrefs [], :text ""} {:varrefs [], :text "woo bar"}]
(generation/expand%
(generation/parse-template "?:[woo bar]"))))
(is (= [{:varrefs [], :text ""} {:varrefs [:woo], :text "WOO"}]
(generation/expand%
(generation/parse-template "?:{woo}")
{:woo "WOO"})))
(is (= ["Hi Tim, What up?"
"Hi Tim, Seeya"
"Hi Tim, WOO! What up?"
"Hi Tim, WOO! Seeya"]
(map :text
(generation/expand%
(generation/parse-template "Hi {name}, ?:[{woo} ][What up?|Seeya]")
{:name "Tim" :woo "WOO!"})))))
(deftest generate-scoring-with-weights
(is (= ["Neighborhood"
"Locality Region Country"]
(map :text
(generation/expand%
(generation/parse-template "[{neighborhood}|{locality} {region} {country}]")
{:neighborhood "Neighborhood"
:locality "Locality"
:region "Region"
:country "Country"}
{:neighborhood 5
:locality 2})))))

View File

@ -0,0 +1,102 @@
(ns lemondronor.circlebot-test
(:require [cljs.test :refer (deftest is testing)]
[lemondronor.circlebot :as circlebot]))
(def epsilon 0.0000001)
(defn a= [a b]
(< (Math/abs (- a b)) epsilon))
;; (deftest bearing->angle
;; (is (angles= (cooleradar/bearing->angle 0) (/ Math/PI 2)))
;; (is (angles= (cooleradar/bearing->angle (/ Math/PI 2)) 0))
;; (is (angles= (cooleradar/bearing->angle Math/PI) (* 2 (/ 3 4) Math/PI))))
(deftest parse-adsbexchange-ac-element
(let [ac {"gnd" "0", "trt" "2", "pos" "1", "call" "", "mil" "0", "ttrk" "",
"dst" "14.6", "reg" "HL7634", "altt" "0", "cou" "South Korea",
"postime" "1575488288571", "galt" "139", "mlat" "0", "spd" "10.5",
"sqk" "", "talt" "", "wtc" "3", "alt" "100", "lon" "-118.416438",
"opicao" "AAR", "interested" "0", "trak" "264.4", "type" "A388",
"trkh" "0", "icao" "71BE34", "lat" "33.937908", "vsit" "1",
"tisb" "0", "vsi" "0", "sat" "0"}]
(is (= (circlebot/parse-adsbexchange-ac-element ac)
{:icao "71BE34"
:registration "HL7634"
:callsign nil
:lon -118.416438
:lat 33.937908
:alt 100
:speed 10.5
:squawk nil
:military? false
:mlat? false
:postime 1575488288571}))))
(deftest prune-history
(let [hist [{:time 0 :id 0} {:time 1000000 :id 1} {:time 2000000 :id 2}]]
(is (= (circlebot/prune-history hist 2500000)
[{:time 2000000 :id 2}]))))
(deftest update-history-db-record
(testing "Updating an existing record"
(let [db {"0" {:icao "0"
:lat 0
:lon 0
:postime 3000
:history [{:time 1000 :id 0}
{:time 2000 :id 1}
{:time 3000 :id 2}]}
"1" :anything}
record {:icao "0"
:lat 1
:lon 1
:postime 3500}]
(is (= (circlebot/update-history-db-record db record)
{"0" {:icao "0"
:lat 1
:lon 1
:postime 3500
:history [{:time 1000 :id 0}
{:time 2000 :id 1}
{:time 3000 :id 2}
{:time 3500 :lat 1 :lon 1}]}
"1" :anything}))))
(testing "Adding a new record"
(let [db {"0" {:icao "0"
:lat 0
:lon 0
:postime 3000
:history [{:time 1000 :id 0}
{:time 2000 :id 1}
{:time 3000 :id 2}]}
"1" :anything}
record {:icao "2"
:lat 1
:lon 1
:postime 3500}]
(is (= (circlebot/update-history-db-record db record)
{"0" {:icao "0"
:lat 0
:lon 0
:postime 3000
:history [{:time 1000 :id 0}
{:time 2000 :id 1}
{:time 3000 :id 2}]}
"1" :anything
"2" {:icao "2"
:lat 1
:lon 1
:postime 3500
:history [{:time 3500 :lat 1 :lon 1}]}})))))
(deftest generation
(let [data {:locality "Palmdale", :continent "North America", :military? true, :alt 3850, :speed "209", :normalized-curviness 14.768651250300287, :accuracy "centroid", :country_a "USA", :continent_gid "whosonfirst:continent:102191575", :name "Palmdale", :squawk "5330", :icao "AE1482", :county_a "LO", :county "Los Angeles County", :source "whosonfirst", :gid "whosonfirst:locality:85923493", :curviness 1269.8089810739468, :locality_gid "whosonfirst:locality:85923493", :region "California", :militaryicao "AE1482", :region_a "CA", :nearbydistance 8.167, :callsign "RAIDR49", :layer "locality", :mlat? false, :country_gid "whosonfirst:country:85633793", :label "Palmdale, CA, USA", :id "85923493", :lon -118.00375, :region_gid "whosonfirst:region:85688637", :lat 34.661074, :militaryregistration "166765", :county_gid "whosonfirst:county:102086957", :started-circling-time 1576266715691, :distance 6.855, :source_id "85923493", :registration "166765", :confidence 0.5, :country "United States", :postime 1576266689756, :nearbylandmark "Living Faith Foursquare Church"}]
(is (re-find #"military" (-> (circlebot/expand-template data) :text))))
(let [data {:locality "Palmdale", :continent "North America", :military? true, :alt 3200, :speed "161", :normalized-curviness 15.783422690487765, :accuracy "centroid", :country_a "USA", :continent_gid "whosonfirst:continent:102191575", :name "Palmdale", :squawk "5330", :icao "AE1482", :county_a "LO", :county "Los Angeles County", :source "whosonfirst", :gid "whosonfirst:locality:85923493", :curviness 1098.803548060181, :locality_gid "whosonfirst:locality:85923493", :region "California", :militaryicao "AE1482", :region_a "CA", :nearbydistance 7.828, :callsign "RAIDR49", :layer "locality", :mlat? false, :country_gid "whosonfirst:country:85633793", :label "Palmdale, CA, USA", :id "85923493", :lon -118.049183, :region_gid "whosonfirst:region:85688637", :lat 34.649808, :militaryregistration "166765", :county_gid "whosonfirst:county:102086957", :started-circling-time 1576267564959, :distance 6.336, :source_id "85923493", :registration "166765", :confidence 0.5, :country "United States", :postime 1576267555709, :nearbylandmark "Living Faith Foursquare Church"}]
(is (re-find #"military" (-> (circlebot/expand-template data) :text))))
)

1
vrs-settings.json Normal file
View File

@ -0,0 +1 @@
{"ver":1,"values":{"VRadarServer-#desktop#-vrsCurrentLocation-default":{"userSuppliedLocation":{"lat":51.45315114582281,"lng":-0.193634033203125},"useBrowserLocation":true,"showCurrentLocation":true},"VRadarServer-#desktop#-vrsMapState-default":{"zoom":12,"mapTypeId":"m","center":{"lat":30.837627249110625,"lng":-117.43440628051759}},"VRadarServer-#desktop#-vrsAircraftListFetcher-default":{"interval":5000,"hideAircraftNotOnMap":true},"VRadarServer-#desktop#-vrsSplitterPosition-default-vrsLayout-A00":{"lengths":[{"name":"S2","pane":1,"vertical":false,"length":659},{"name":"S1","pane":2,"vertical":true,"length":650}]},"VRadarServer-#desktop#-vrsAircraftAutoSelect-default":{"enabled":false,"selectClosest":true,"offRadarAction":"---","filters":[]},"VRadarServer-#desktop#-vrsAircraftPlotterOptions-default":{"showAltitudeStalk":false,"suppressAltitudeStalkWhenZoomedOut":true,"showPinText":true,"pinTexts":["reg","csn","alt","---","---","---"],"pinTextLines":3,"hideEmptyPinTextLines":true,"trailDisplay":"b","trailType":"b","showRangeCircles":true,"rangeCircleInterval":10,"rangeCircleDistanceUnit":"nm","rangeCircleCount":6,"rangeCircleOddColour":"#333333","rangeCircleOddWeight":1,"rangeCircleEvenColour":"#111111","rangeCircleEvenWeight":2,"onlyUsePre22Icons":false,"aircraftMarkerClustererMaxZoom":5},"VRadarServer-#desktop#-vrsAircraftDetailPlugin-default":{"showUnits":true,"items":["alt","aty","vsi","spd","hdg","bng","dis","lat","lng","sqk","eng","spc","wtc","trt","avs","sig","mlt","tsb","mct","tim","fct","opi","ser","yrb","int","tag","pic"],"useShortLabels":false},"VRadarServer-#desktop#-vrsAircraftSorter-default":{"sortFields":[{"field":"mod","ascending":true},{"field":"---","ascending":true},{"field":"---","ascending":true}],"showEmergencySquawks":1,"showInteresting":1},"VRadarServer-#desktop#-vrsAircraftListPlugin-default":{"columns":["opf","sil","typ","csn","reg","alt","spd","ico","mlt","mil","cou"],"showUnits":true}}}