From a62fd3964e73afd3868fa6a6750ab8a70b5db845 Mon Sep 17 00:00:00 2001 From: John Wiseman Date: Mon, 22 Jun 2020 09:23:07 -0700 Subject: [PATCH] Switched from basestation.sqb to mictronics db. --- advisorycircular.sh | 4 +- install.sh | 57 ++++++++------ src/main/lemondronor/advisorycircular.cljs | 88 ++++++++++++++++------ 3 files changed, 101 insertions(+), 48 deletions(-) diff --git a/advisorycircular.sh b/advisorycircular.sh index 7ebdecd..3221cb6 100644 --- a/advisorycircular.sh +++ b/advisorycircular.sh @@ -5,10 +5,10 @@ dir=$(dirname "$0") instance="$1" config_dir=/etc/advisorycircular run_dir=/var/lib/advisorycircular -sqb_path=/usr/local/share/advisorycircular/clean-basestation.sqb +ac_db_path=/usr/local/share/advisorycircular/aircraft-info/aircraft-info.sqb interval=15 config_path="$config_dir"/"$instance"-config.yaml secrets_path="$config_dir"/"$instance"-secrets.yaml history_path="$run_dir"/"$instance"-history.json -(cd "$run_dir" && "$dir"/intervalexec $interval node $dir/out/advisorycircular.js --config "$config_path" --secrets "$secrets_path" --basestation-sqb "$sqb_path" --history "$history_path" --log-prefix "$instance") +(cd "$run_dir" && "$dir"/intervalexec $interval node $dir/out/advisorycircular.js --config "$config_path" --secrets "$secrets_path" --aircraft-info-db "$ac_db_path" --history "$history_path" --log-prefix "$instance") diff --git a/install.sh b/install.sh index 3811776..35b3645 100755 --- a/install.sh +++ b/install.sh @@ -1,41 +1,54 @@ #!/bin/bash set -e -install_path=/usr/local/share/advisorycircular -config_path=/etc/advisorycircular -run_path=/var/lib/advisorycircular +install_dir=/usr/local/share/advisorycircular +config_dir=/etc/advisorycircular +run_dir=/var/lib/advisorycircular user=advisorycircular # Create install, config, and run directories if they doesn't exist. -sudo mkdir -p "$install_path" -sudo mkdir -p "$config_path" -sudo mkdir -p "$run_path" +echo Creating directories +mkdir -p "$install_dir" +mkdir -p "$config_dir" +mkdir -p "$run_dir" # Create advisory-circular user if it doesn't exist. if ! id -u ${user} >/dev/null 2>&1 then - sudo adduser --system --home ${install_path} --no-create-home --quiet ${user} + echo Creating ${user} user + adduser --system --home ${install_dir} --no-create-home --quiet ${user} fi # Compile. +echo Compiling npm install npx shadow-cljs compile script # Install -sudo install intervalexec.py "$install_path"/intervalexec -sudo install advisorycircular.sh "$install_path"/advisorycircular -sudo cp -r node_modules "$install_path" -sudo cp -r .shadow-cljs "$install_path" -sudo mkdir -p "$install_path"/out -sudo cp out/advisorycircular.js "$install_path"/out -sudo cp clean-basestation.sqb "$install_path" -sudo chown -R "$user" "$install_path" +echo Copying files +install intervalexec.py "$install_dir"/intervalexec +install advisorycircular.sh "$install_dir"/advisorycircular +cp -r node_modules "$install_dir" +cp -r .shadow-cljs "$install_dir" +mkdir -p "$install_dir"/out +cp out/advisorycircular.js "$install_dir"/out +cp clean-basestation.sqb "$install_dir" -sudo cp config-skel.yaml "$config_path" -sudo cp secrets-skel.yaml "$config_path" -sudo chmod go-r "$config_path"/secrets-skel.yaml -sudo chown -R "$user" "$config_path" +cp config-skel.yaml "$config_dir" +cp secrets-skel.yaml "$config_dir" -sudo chown -R "$user" "$run_path" -sudo chmod go-w "$run_path" +cp advisorycircular@.service /lib/systemd/system -sudo cp advisorycircular@.service /lib/systemd/system +# Download aircraft info DB. +echo Downloading aircraft info database +mkdir -p "$install_dir"/aircraft-info +curl 'https://www.mictronics.de/aircraft-database/indexedDB.php' > "$install_dir"/aircraft-info/db.zip +(cd "$install_dir"/aircraft-info && unzip -o db.zip) +# Convert JSON to sqlite DB. +node "$install_dir"/out/advisorycircular.js --aircraft-info-db /tmp/aircraft-info-$$.sqb --create-aircraft-info-db-from-json "$install_dir"/aircraft-info/aircrafts.json && mv /tmp/aircraft-info-$$.sqb "$install_dir"/aircraft-info/aircraft-info.sqb + +chown -R "$user" "$install_dir" +chown -R "$user" "$config_dir" +chown -R "$user" "$run_dir" + +chmod go-r "$config_dir"/secrets-skel.yaml +chmod go-w "$run_dir" diff --git a/src/main/lemondronor/advisorycircular.cljs b/src/main/lemondronor/advisorycircular.cljs index 0c67a7c..ec99dcb 100644 --- a/src/main/lemondronor/advisorycircular.cljs +++ b/src/main/lemondronor/advisorycircular.cljs @@ -3,9 +3,9 @@ ["commander" :as commander] ["fs" :as fs] [cljs.pprint :as pprint] - [cljs.reader :as reader] [clojure.set :as set] [clojure.string :as string] + [goog.object] [kitchen-async.promise :as p] [lemondronor.advisorycircular.adsbx :as adsbx] [lemondronor.advisorycircular.generation :as generation] @@ -27,15 +27,52 @@ (.parse js/JSON s)) -(defn get-basestation-sqb-record [icao db-path] + +(defn create-aircraft-info-db [json-path sqb-path] + (log-info "Reading %s" json-path) + (p/let [info-str (util/read-file json-path) + info (parse-json info-str) + _ (log-info "Creating DB %s" sqb-path) + db (sqlite/open sqb-path clj->js {js/Promise js/Promise})] + (.run db "DROP TABLE IF EXISTS aircraft") + (.run db "DROP INDEX IF EXISTS idx_aircraft_icao") + (.run db + (str "CREATE TABLE aircraft (" + "icao TEXT NOT NULL PRIMARY KEY," + "registration TEXT," + "type TEXT" + ");")) + (.run db "BEGIN TRANSACTION") + (p/let [count (p/loop [count 0 + keys (seq (goog.object/getKeys info)) + _ nil] + (when (and (> count 0) (zero? (mod count 100000))) + (log-info "Inserted %s records" count)) + (if (seq keys) + (let [icao (first keys) + rec (aget info icao) + reg (aget rec "r") + type (aget rec "d")] + (p/recur + (inc count) + (rest keys) + (.run db "INSERT INTO aircraft (icao, registration, type) VALUES (?, ?, ?)" + icao + (aget rec "r") + (aget rec "d")))) + count))] + (.run db "CREATE UNIQUE INDEX idx_aircraft_icao ON aircraft (icao)") + (log-info "Committing %s records" count)) + (.run db "COMMIT"))) + +(defn get-aircraft-info-record [icao db-path] (log-info "%s: Looking up in %s" icao db-path) (p/let [record - (p/-> (sqlite/open db-path clj->js { js/Promise js/Promise }) - (.get "SELECT Registration, Type, RegisteredOwners from Aircraft where ModeS = ?" icao))] - (log-info "%s: basestation.sqb: %s" icao (js->clj record :keywordize-keys true)) + (p/-> (sqlite/open db-path clj->js {js/Promise js/Promise}) + (.get "SELECT registration, type from aircraft where icao = ?" icao))] + (log-info "%s: aircraft-info record: %s" icao (js->clj record :keywordize-keys true)) (-> record - (js->clj :keywordize-keys true) - (set/rename-keys {:Registration :registration :Type :type :RegisteredOwners :registered-owners})))) + (js->clj :keywordize-keys true)))) (defn parse-adsbexchange-ac-element [e] @@ -418,29 +455,29 @@ (.toFixed n d)) -(defn merge-adsbx-sqb [ac sqb] +(defn merge-adsbx-aircraft-db-rec [ac ac-db-rec] (cond-> ac - (and (nil? (:registration ac)) (:registration sqb)) - (assoc :registration (:registration sqb)) + (and (nil? (:registration ac)) (:registration ac-db-rec)) + (assoc :registration (:registration ac-db-rec)) true - (assoc :type (:type sqb)))) + (assoc :type (:type ac-db-rec)))) ;; Creates a template expansion map from the following: ;; ;; * ac - The ADSBX API entry -;; * sqb - The baseseation.sqb record +;; * ac-db-rec - The aircraft DB record ;; * reverse - reverse geocoder record ;; * wiki-nearby - nearby landmarks w/ Wikipedia pages ;; * nearby - nearby landmarks. -(defn template-data [ac sqb reverse nearby] +(defn template-data [ac ac-db-rec reverse nearby] (let [rev-props (:properties reverse) nearby (:properties nearby) info (cond-> (-> ac (dissoc :history :type) (merge rev-props) - (merge-adsbx-sqb sqb)) + (merge-adsbx-aircraft-db-rec ac-db-rec)) (:military? ac) (-> (assoc :militaryregistration (:registration ac) :militaryicao (:icao ac))) @@ -457,8 +494,8 @@ info)) -(defn generate-description [ac sqb reverse nearby] - (let [info (template-data ac sqb reverse nearby) +(defn generate-description [ac ac-db-rec reverse nearby] + (let [info (template-data ac ac-db-rec reverse nearby) expansion (expand-template info)] (when (not expansion) (log-warn "Info: %s" info)) @@ -581,9 +618,9 @@ (aircraft-photo (:icao ac) (:registration ac)) (p/let [nearby (landmark config lat lon) ;;_ (log-info "WOOO %s" nearby) - sqb (if-let [sqb-path (:basestation-sqb config)] - (get-basestation-sqb-record icao sqb-path))] - (let [description (generate-description ac sqb coarse nearby)] + ac-db-rec (if-let [ac-info-db-path (:aircraft-info-db-path config)] + (get-aircraft-info-record icao ac-info-db-path))] + (let [description (generate-description ac ac-db-rec coarse nearby)] (log-info "%s Description: %s" (:icao ac) description) description))]) (fn [[screenshot-path ac-photo description]] @@ -634,7 +671,7 @@ ;; training. :minimum-airport-distance-km 2.5 :history-db-path "advisorycircular.json" - :basestation-sqb "basestation.sqb" + :aircraft-info-db-path "aircraft-info.sqb" :twitter {:enabled? true}}) @@ -650,8 +687,8 @@ (assoc-in [:twitter :enabled?] (.-tweeting commander)) (.-peliasUrl commander) (assoc-in [:pelias :url] (.-peliasUrl commander)) - (.-basestationSqb commander) - (assoc :basestation-sqb (.-basestationSqb commander)) + (.-aircraftInfoDb commander) + (assoc :aircraft-info-db-path (.-aircraftInfoDb commander)) (.-history commander) (assoc :history-db-path (.-history commander)) (.-lat commander) @@ -670,7 +707,7 @@ (let [required [[:lat] [:lon] [:radius-km] - [:basestation-sqb] + [:aircraft-info-db-path] [:adsbx :url] [:adsbx :api-key] [:pelias :url]] @@ -696,7 +733,7 @@ (.option "--adsbx-url " "ADSBX API url") (.option "--pelias-url " "Base pelias geocoder URL") (.option "--radius " "Radius of the circle of interest, in km" parse-number) - (.option "--basestation-sqb " "Path to a basestation.sqb database file") + (.option "--aircraft-info-db " "Path to an aircraft info DB file") (.option "--tweeting" "Enables tweeting") (.option "--no-tweeting" "Do not tweet") (.option "--config " "Path to the configuration yaml file") @@ -704,6 +741,7 @@ (.option "--history " "Path to history/state file" "advisorycircular.json") (.option "--log-prefix " "Log prefix to use") (.option "--airport-geojson" "Generate airport GEOJSON and exit") + (.option "--create-aircraft-info-db-from-json " "Generate aircraft info DB and exit") (.option "--test") (.option "--icao ") @@ -713,6 +751,8 @@ (reset! log-prefix (or (.-logPrefix commander) "")) (p/try (cond + (.-createAircraftInfoDbFromJson commander) + (create-aircraft-info-db (.-createAircraftInfoDbFromJson commander) (.-aircraftInfoDb commander)) (.-airportGeojson commander) (p/let [base-config (if-let [config-path (.-config commander)] (util/read-config config-path)