tyrel/advisory-circular
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Switched from basestation.sqb to mictronics db.

Browse Source
This commit is contained in:
John Wiseman 2020-06-22 09:23:07 -07:00
parent 663f954410
commit a62fd3964e
3 changed files with 101 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
advisorycircular.sh
View File

@ -5,10 +5,10 @@ dir=$(dirname "$0")
instance="$1" instance="$1"
config_dir=/etc/advisorycircular config_dir=/etc/advisorycircular
run_dir=/var/lib/advisorycircular run_dir=/var/lib/advisorycircular
sqb_path=/usr/local/share/advisorycircular/clean-basestation.sqb ac_db_path=/usr/local/share/advisorycircular/aircraft-info/aircraft-info.sqb
interval=15 interval=15
config_path="$config_dir"/"$instance"-config.yaml config_path="$config_dir"/"$instance"-config.yaml
secrets_path="$config_dir"/"$instance"-secrets.yaml secrets_path="$config_dir"/"$instance"-secrets.yaml
history_path="$run_dir"/"$instance"-history.json history_path="$run_dir"/"$instance"-history.json
(cd "$run_dir" && "$dir"/intervalexec $interval node $dir/out/advisorycircular.js --config "$config_path" --secrets "$secrets_path" --basestation-sqb "$sqb_path" --history "$history_path" --log-prefix "$instance") (cd "$run_dir" && "$dir"/intervalexec $interval node $dir/out/advisorycircular.js --config "$config_path" --secrets "$secrets_path" --aircraft-info-db "$ac_db_path" --history "$history_path" --log-prefix "$instance")

57
install.sh
View File

@ -1,41 +1,54 @@
#!/bin/bash #!/bin/bash
set -e set -e
install_path=/usr/local/share/advisorycircular install_dir=/usr/local/share/advisorycircular
config_path=/etc/advisorycircular config_dir=/etc/advisorycircular
run_path=/var/lib/advisorycircular run_dir=/var/lib/advisorycircular
user=advisorycircular user=advisorycircular
# Create install, config, and run directories if they doesn't exist. # Create install, config, and run directories if they doesn't exist.
sudo mkdir -p "$install_path" echo Creating directories
sudo mkdir -p "$config_path" mkdir -p "$install_dir"
sudo mkdir -p "$run_path" mkdir -p "$config_dir"
mkdir -p "$run_dir"
# Create advisory-circular user if it doesn't exist. # Create advisory-circular user if it doesn't exist.
if ! id -u ${user} >/dev/null 2>&1 if ! id -u ${user} >/dev/null 2>&1
then then
sudo adduser --system --home ${install_path} --no-create-home --quiet ${user} echo Creating ${user} user
adduser --system --home ${install_dir} --no-create-home --quiet ${user}
fi fi
# Compile. # Compile.
echo Compiling
npm install npm install
npx shadow-cljs compile script npx shadow-cljs compile script
# Install # Install
sudo install intervalexec.py "$install_path"/intervalexec echo Copying files
sudo install advisorycircular.sh "$install_path"/advisorycircular install intervalexec.py "$install_dir"/intervalexec
sudo cp -r node_modules "$install_path" install advisorycircular.sh "$install_dir"/advisorycircular
sudo cp -r .shadow-cljs "$install_path" cp -r node_modules "$install_dir"
sudo mkdir -p "$install_path"/out cp -r .shadow-cljs "$install_dir"
sudo cp out/advisorycircular.js "$install_path"/out mkdir -p "$install_dir"/out
sudo cp clean-basestation.sqb "$install_path" cp out/advisorycircular.js "$install_dir"/out
sudo chown -R "$user" "$install_path" cp clean-basestation.sqb "$install_dir"
sudo cp config-skel.yaml "$config_path" cp config-skel.yaml "$config_dir"
sudo cp secrets-skel.yaml "$config_path" cp secrets-skel.yaml "$config_dir"
sudo chmod go-r "$config_path"/secrets-skel.yaml
sudo chown -R "$user" "$config_path"
sudo chown -R "$user" "$run_path" cp advisorycircular@.service /lib/systemd/system
sudo chmod go-w "$run_path"
sudo cp advisorycircular@.service /lib/systemd/system # Download aircraft info DB.
echo Downloading aircraft info database
mkdir -p "$install_dir"/aircraft-info
curl 'https://www.mictronics.de/aircraft-database/indexedDB.php' > "$install_dir"/aircraft-info/db.zip
(cd "$install_dir"/aircraft-info && unzip -o db.zip)
# Convert JSON to sqlite DB.
node "$install_dir"/out/advisorycircular.js --aircraft-info-db /tmp/aircraft-info-$$.sqb --create-aircraft-info-db-from-json "$install_dir"/aircraft-info/aircrafts.json && mv /tmp/aircraft-info-$$.sqb "$install_dir"/aircraft-info/aircraft-info.sqb
chown -R "$user" "$install_dir"
chown -R "$user" "$config_dir"
chown -R "$user" "$run_dir"
chmod go-r "$config_dir"/secrets-skel.yaml
chmod go-w "$run_dir"

88
src/main/lemondronor/advisorycircular.cljs
View File

@ -3,9 +3,9 @@
["commander" :as commander] ["commander" :as commander]
["fs" :as fs] ["fs" :as fs]
[cljs.pprint :as pprint] [cljs.pprint :as pprint]
[cljs.reader :as reader]
[clojure.set :as set] [clojure.set :as set]
[clojure.string :as string] [clojure.string :as string]
[goog.object]
[kitchen-async.promise :as p] [kitchen-async.promise :as p]
[lemondronor.advisorycircular.adsbx :as adsbx] [lemondronor.advisorycircular.adsbx :as adsbx]
[lemondronor.advisorycircular.generation :as generation] [lemondronor.advisorycircular.generation :as generation]
@ -27,15 +27,52 @@
(.parse js/JSON s)) (.parse js/JSON s))
(defn get-basestation-sqb-record [icao db-path]
(defn create-aircraft-info-db [json-path sqb-path]
(log-info "Reading %s" json-path)
(p/let [info-str (util/read-file json-path)
info (parse-json info-str)
_ (log-info "Creating DB %s" sqb-path)
db (sqlite/open sqb-path clj->js {js/Promise js/Promise})]
(.run db "DROP TABLE IF EXISTS aircraft")
(.run db "DROP INDEX IF EXISTS idx_aircraft_icao")
(.run db
(str "CREATE TABLE aircraft ("
"icao TEXT NOT NULL PRIMARY KEY,"
"registration TEXT,"
"type TEXT"
");"))
(.run db "BEGIN TRANSACTION")
(p/let [count (p/loop [count 0
keys (seq (goog.object/getKeys info))
_ nil]
(when (and (> count 0) (zero? (mod count 100000)))
(log-info "Inserted %s records" count))
(if (seq keys)
(let [icao (first keys)
rec (aget info icao)
reg (aget rec "r")
type (aget rec "d")]
(p/recur
(inc count)
(rest keys)
(.run db "INSERT INTO aircraft (icao, registration, type) VALUES (?, ?, ?)"
icao
(aget rec "r")
(aget rec "d"))))
count))]
(.run db "CREATE UNIQUE INDEX idx_aircraft_icao ON aircraft (icao)")
(log-info "Committing %s records" count))
(.run db "COMMIT")))
(defn get-aircraft-info-record [icao db-path]
(log-info "%s: Looking up in %s" icao db-path) (log-info "%s: Looking up in %s" icao db-path)
(p/let [record (p/let [record
(p/-> (sqlite/open db-path clj->js { js/Promise js/Promise }) (p/-> (sqlite/open db-path clj->js {js/Promise js/Promise})
(.get "SELECT Registration, Type, RegisteredOwners from Aircraft where ModeS = ?" icao))] (.get "SELECT registration, type from aircraft where icao = ?" icao))]
(log-info "%s: basestation.sqb: %s" icao (js->clj record :keywordize-keys true)) (log-info "%s: aircraft-info record: %s" icao (js->clj record :keywordize-keys true))
(-> record (-> record
(js->clj :keywordize-keys true) (js->clj :keywordize-keys true))))
(set/rename-keys {:Registration :registration :Type :type :RegisteredOwners :registered-owners}))))
(defn parse-adsbexchange-ac-element [e] (defn parse-adsbexchange-ac-element [e]
@ -418,29 +455,29 @@
(.toFixed n d)) (.toFixed n d))
(defn merge-adsbx-sqb [ac sqb] (defn merge-adsbx-aircraft-db-rec [ac ac-db-rec]
(cond-> ac (cond-> ac
(and (nil? (:registration ac)) (:registration sqb)) (and (nil? (:registration ac)) (:registration ac-db-rec))
(assoc :registration (:registration sqb)) (assoc :registration (:registration ac-db-rec))
true true
(assoc :type (:type sqb)))) (assoc :type (:type ac-db-rec))))
;; Creates a template expansion map from the following: ;; Creates a template expansion map from the following:
;; ;;
;; * ac - The ADSBX API entry ;; * ac - The ADSBX API entry
;; * sqb - The baseseation.sqb record ;; * ac-db-rec - The aircraft DB record
;; * reverse - reverse geocoder record ;; * reverse - reverse geocoder record
;; * wiki-nearby - nearby landmarks w/ Wikipedia pages ;; * wiki-nearby - nearby landmarks w/ Wikipedia pages
;; * nearby - nearby landmarks. ;; * nearby - nearby landmarks.
(defn template-data [ac sqb reverse nearby] (defn template-data [ac ac-db-rec reverse nearby]
(let [rev-props (:properties reverse) (let [rev-props (:properties reverse)
nearby (:properties nearby) nearby (:properties nearby)
info (cond-> (-> ac info (cond-> (-> ac
(dissoc :history :type) (dissoc :history :type)
(merge rev-props) (merge rev-props)
(merge-adsbx-sqb sqb)) (merge-adsbx-aircraft-db-rec ac-db-rec))
(:military? ac) (:military? ac)
(-> (assoc :militaryregistration (:registration ac) (-> (assoc :militaryregistration (:registration ac)
:militaryicao (:icao ac))) :militaryicao (:icao ac)))
@ -457,8 +494,8 @@
info)) info))
(defn generate-description [ac sqb reverse nearby] (defn generate-description [ac ac-db-rec reverse nearby]
(let [info (template-data ac sqb reverse nearby) (let [info (template-data ac ac-db-rec reverse nearby)
expansion (expand-template info)] expansion (expand-template info)]
(when (not expansion) (when (not expansion)
(log-warn "Info: %s" info)) (log-warn "Info: %s" info))
@ -581,9 +618,9 @@
(aircraft-photo (:icao ac) (:registration ac)) (aircraft-photo (:icao ac) (:registration ac))
(p/let [nearby (landmark config lat lon) (p/let [nearby (landmark config lat lon)
;;_ (log-info "WOOO %s" nearby) ;;_ (log-info "WOOO %s" nearby)
sqb (if-let [sqb-path (:basestation-sqb config)] ac-db-rec (if-let [ac-info-db-path (:aircraft-info-db-path config)]
(get-basestation-sqb-record icao sqb-path))] (get-aircraft-info-record icao ac-info-db-path))]
(let [description (generate-description ac sqb coarse nearby)] (let [description (generate-description ac ac-db-rec coarse nearby)]
(log-info "%s Description: %s" (:icao ac) description) (log-info "%s Description: %s" (:icao ac) description)
description))]) description))])
(fn [[screenshot-path ac-photo description]] (fn [[screenshot-path ac-photo description]]
@ -634,7 +671,7 @@
;; training. ;; training.
:minimum-airport-distance-km 2.5 :minimum-airport-distance-km 2.5
:history-db-path "advisorycircular.json" :history-db-path "advisorycircular.json"
:basestation-sqb "basestation.sqb" :aircraft-info-db-path "aircraft-info.sqb"
:twitter {:enabled? true}}) :twitter {:enabled? true}})
@ -650,8 +687,8 @@
(assoc-in [:twitter :enabled?] (.-tweeting commander)) (assoc-in [:twitter :enabled?] (.-tweeting commander))
(.-peliasUrl commander) (.-peliasUrl commander)
(assoc-in [:pelias :url] (.-peliasUrl commander)) (assoc-in [:pelias :url] (.-peliasUrl commander))
(.-basestationSqb commander) (.-aircraftInfoDb commander)
(assoc :basestation-sqb (.-basestationSqb commander)) (assoc :aircraft-info-db-path (.-aircraftInfoDb commander))
(.-history commander) (.-history commander)
(assoc :history-db-path (.-history commander)) (assoc :history-db-path (.-history commander))
(.-lat commander) (.-lat commander)
@ -670,7 +707,7 @@
(let [required [[:lat] (let [required [[:lat]
[:lon] [:lon]
[:radius-km] [:radius-km]
[:basestation-sqb] [:aircraft-info-db-path]
[:adsbx :url] [:adsbx :url]
[:adsbx :api-key] [:adsbx :api-key]
[:pelias :url]] [:pelias :url]]
@ -696,7 +733,7 @@
(.option "--adsbx-url <url>" "ADSBX API url") (.option "--adsbx-url <url>" "ADSBX API url")
(.option "--pelias-url <url>" "Base pelias geocoder URL") (.option "--pelias-url <url>" "Base pelias geocoder URL")
(.option "--radius <radius>" "Radius of the circle of interest, in km" parse-number) (.option "--radius <radius>" "Radius of the circle of interest, in km" parse-number)
(.option "--basestation-sqb <path>" "Path to a basestation.sqb database file") (.option "--aircraft-info-db <path>" "Path to an aircraft info DB file")
(.option "--tweeting" "Enables tweeting") (.option "--tweeting" "Enables tweeting")
(.option "--no-tweeting" "Do not tweet") (.option "--no-tweeting" "Do not tweet")
(.option "--config <path>" "Path to the configuration yaml file") (.option "--config <path>" "Path to the configuration yaml file")
@ -704,6 +741,7 @@
(.option "--history <path>" "Path to history/state file" "advisorycircular.json") (.option "--history <path>" "Path to history/state file" "advisorycircular.json")
(.option "--log-prefix <prefix>" "Log prefix to use") (.option "--log-prefix <prefix>" "Log prefix to use")
(.option "--airport-geojson" "Generate airport GEOJSON and exit") (.option "--airport-geojson" "Generate airport GEOJSON and exit")
(.option "--create-aircraft-info-db-from-json <json path>" "Generate aircraft info DB and exit")
(.option "--test") (.option "--test")
(.option "--icao <icao>") (.option "--icao <icao>")
@ -713,6 +751,8 @@
(reset! log-prefix (or (.-logPrefix commander) "")) (reset! log-prefix (or (.-logPrefix commander) ""))
(p/try (p/try
(cond (cond
(.-createAircraftInfoDbFromJson commander)
(create-aircraft-info-db (.-createAircraftInfoDbFromJson commander) (.-aircraftInfoDb commander))
(.-airportGeojson commander) (.-airportGeojson commander)
(p/let [base-config (if-let [config-path (.-config commander)] (p/let [base-config (if-let [config-path (.-config commander)]
(util/read-config config-path) (util/read-config config-path)