From 071ef7e91aec11fd3be5fe89460c535009d9478d Mon Sep 17 00:00:00 2001 From: Tyrel Souza <923113+tyrelsouza@users.noreply.github.com> Date: Tue, 25 Feb 2020 22:34:17 -0500 Subject: [PATCH] mock so i dont keep hitting noaa --- go.mod | 5 ++-- go.sum | 25 +++++++++++--------- main.go | 35 ++++------------------------ noaa_client/noaa_client.go | 41 +++++++++++++++++++++++++++++++++ noaa_client/noaa_client_test.go | 36 +++++++++++++++++++++++++++++ test/KeeneResponse.html | 39 +++++++++++++++++++++++++++++++ 6 files changed, 137 insertions(+), 44 deletions(-) create mode 100644 noaa_client/noaa_client.go create mode 100644 noaa_client/noaa_client_test.go create mode 100644 test/KeeneResponse.html diff --git a/go.mod b/go.mod index 6e60f56..a02ab07 100644 --- a/go.mod +++ b/go.mod @@ -3,8 +3,7 @@ module gitlab/tyrelsouza/gfsmos go 1.13 require ( - github.com/antchfx/xpath v1.1.4 - github.com/antchfx/xquery v0.0.0-20180515051857-ad5b8c7a47b0 + github.com/stretchr/testify v1.5.1 github.com/urfave/cli/v2 v2.1.1 - golang.org/x/net v0.0.0-20200222125558-5a598a2470a0 + gopkg.in/h2non/gock.v1 v1.0.15 ) diff --git a/go.sum b/go.sum index b3bbf02..7bd19e6 100644 --- a/go.sum +++ b/go.sum @@ -1,23 +1,26 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/antchfx/xpath v1.1.4 h1:naPIpjBGeT3eX0Vw7E8iyHsY8FGt6EbGdkcd8EZCo+g= -github.com/antchfx/xpath v1.1.4/go.mod h1:Yee4kTMuNiPYJ7nSNorELQMr1J33uOpXDMByNYhvtNk= -github.com/antchfx/xquery v0.0.0-20180515051857-ad5b8c7a47b0 h1:JaCC8jz0zdMLk2m+qCCVLLLM/PL93p84w4pK3aJWj60= -github.com/antchfx/xquery v0.0.0-20180515051857-ad5b8c7a47b0/go.mod h1:LzD22aAzDP8/dyiCKFp31He4m2GPjl0AFyzDtZzUu9M= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 h1:2VTzZjLZBgl62/EtslCrtky5vbi9dd7HrQPQIx6wqiw= +github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542/go.mod h1:Ow0tF8D4Kplbc8s8sSb3V2oUCygFHVp8gC3Dn6U4MNI= +github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32 h1:W6apQkHrMkS0Muv8G/TipAy/FJl/rCYT0+EuS8+Z0z4= +github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32/go.mod h1:9wM+0iRr9ahx58uYLpLIr5fm8diHn0JbqRycJi6w0Ms= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= -github.com/urfave/cli v1.22.2 h1:gsqYFH8bb9ekPA12kRo0hfjngWQjkJPlN9R0N78BoUo= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/urfave/cli/v2 v2.1.1 h1:Qt8FeAtxE/vfdrLmR3rxR6JRE0RoVmbXu8+6kZtYU4k= github.com/urfave/cli/v2 v2.1.1/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/net v0.0.0-20200222125558-5a598a2470a0 h1:MsuvTghUPjX762sGLnGsxC3HM0B5r83wEtYcYR8/vRs= -golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/h2non/gock.v1 v1.0.15 h1:SzLqcIlb/fDfg7UvukMpNcWsu7sI5tWwL+KCATZqks0= +gopkg.in/h2non/gock.v1 v1.0.15/go.mod h1:sX4zAkdYX1TRGJ2JY156cFspQn4yRWn6p9EMdODlynE= +gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/main.go b/main.go index 106a12c..ac65450 100644 --- a/main.go +++ b/main.go @@ -2,11 +2,8 @@ package main import ( "fmt" - "github.com/antchfx/xpath" - "github.com/antchfx/xquery/html" - "io/ioutil" + "gitlab/tyrelsouza/gfsmos/noaa_client" "log" - "net/http" "os" "strings" @@ -14,6 +11,7 @@ import ( ) func main() { + app := &cli.App{ Name: "GFS MOS", Usage: "Parse NOAA's GFS MOS for a given airport", @@ -22,8 +20,9 @@ func main() { if airport == "" { return cli.NewExitError("please pass the airport's ICAO as an argument", 1) } - pre := getDataFromPre(airport) - fmt.Println(pre) + api := noaa_client.New(airport) + pre := api.GetDataRaw() + fmt.Print(pre) return nil }, } @@ -33,27 +32,3 @@ func main() { log.Fatal(err) } } - -func getDataFromPre(airport string) string { - url := fmt.Sprintf("https://www.nws.noaa.gov/cgi-bin/mos/getmav.pl?sta=%s", airport) - - response, err := http.Get(url) - defer response.Body.Close() - if err != nil { - log.Fatal(err) - } - - bytes, err := ioutil.ReadAll(response.Body) - if err != nil { - log.Fatal(err) - } - doc, err := htmlquery.Parse(string(bytes)) - if err != nil{ - panic(err) - } - for _, n := range htmlquery.Find(doc, "//a/@href") { - fmt.Printf("%s \n", htmlquery.SelectAttr(n, "href")) // output href - } - - return string(bytes) -} diff --git a/noaa_client/noaa_client.go b/noaa_client/noaa_client.go new file mode 100644 index 0000000..beecaa4 --- /dev/null +++ b/noaa_client/noaa_client.go @@ -0,0 +1,41 @@ +package noaa_client + +import ( + "fmt" + "io/ioutil" + "net/http" + "regexp" + "strings" +) + +type API struct { + Airport string +} + +func New(airport string) *API { + return &API{airport} +} + +func (api *API) GetBody(url string) ([]byte, error) { + resp, err := http.Get(url) + if err != nil { + return nil, err + } + defer resp.Body.Close() + body, err := ioutil.ReadAll(resp.Body) + // handling error and doing stuff with body that needs to be unit tested + return body, err +} + +func (api *API) GetDataRaw() string { + url := fmt.Sprintf("https://www.nws.noaa.gov/cgi-bin/mos/getmav.pl?sta=%s", api.Airport) + body, _ := api.GetBody(url) + + re := regexp.MustCompile(`(?s)(.*)`) + + for _, element := range re.FindAllStringSubmatch(string(body), -1) { + return strings.TrimSpace(element[1]) + } + + return "" +} \ No newline at end of file diff --git a/noaa_client/noaa_client_test.go b/noaa_client/noaa_client_test.go new file mode 100644 index 0000000..19943e0 --- /dev/null +++ b/noaa_client/noaa_client_test.go @@ -0,0 +1,36 @@ +package noaa_client + +import ( + "github.com/stretchr/testify/assert" + "gopkg.in/h2non/gock.v1" + "io/ioutil" + "log" + "testing" +) +func fakeResponse() string { + b, err := ioutil.ReadFile("../test/KeeneResponse.html") // b has type []byte + if err != nil { + log.Fatal(err) + } + return string(b) +} + + +func TestFoo(t *testing.T) { + defer gock.Off() // Flush pending mocks after test execution + + gock.New("https://www.nws.noaa.gov"). + Get("/cgi-bin/mos/getmav.pl"). + MatchParam("sta", "KEEN"). + Reply(200). + BodyString(fakeResponse()) + + // Use Client & URL from our local test server + api := API{ + "KEEN", + } + + body := api.GetDataRaw() + + assert.Equal(t, "bob", body) +} \ No newline at end of file diff --git a/test/KeeneResponse.html b/test/KeeneResponse.html new file mode 100644 index 0000000..21f06df --- /dev/null +++ b/test/KeeneResponse.html @@ -0,0 +1,39 @@ + +GFS MOS FORECASTS +

GFS MOS FORECASTS

+
+
+ KEEN   GFS MOS GUIDANCE    2/25/2020  1800 UTC
+ DT /FEB  26                  /FEB  27                /FEB  28
+ HR   00 03 06 09 12 15 18 21 00 03 06 09 12 15 18 21 00 03 06 12 18
+ N/X              34          43          38          47       20
+ TMP  42 39 37 37 36 38 41 41 40 40 41 41 41 47 45 41 37 31 26 23 33
+ DPT  33 34 33 32 31 32 34 35 33 33 34 34 34 35 29 22 19 13 11 10  6
+ CLD  OV OV OV OV OV OV OV OV OV OV OV OV OV OV OV OV BK BK BK BK OV
+ WDR  02 01 01 01 02 04 02 04 04 07 04 04 06 20 23 23 22 22 22 24 25
+ WSP  01 01 01 01 01 01 01 03 02 06 08 18 16 19 14 10 09 11 12 14 17
+ P06        43    53    59    38    36    92    87    21    14 11 13
+ P12              60          59          98          93       19
+ Q06         0     1     1     0     0     5     4     0     0  0  0
+ Q12               1           1           4           4        0
+ T06      0/ 0  0/ 0  0/ 0  0/ 0  0/ 0  0/ 7  1/ 6  1/11  0/ 9  0/ 3
+ T12                  0/ 0        0/ 0        1/ 8        1/16  0/ 5
+ POZ   0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  1  1  3  1
+ POS   0  9  7  5  7  7  7  6  5  3  1  4  3 16 37 74 91 98 98 98 99
+ TYP   R  R  R  R  R  R  R  R  R  R  R  R  R  R  R  S  S  S  S  S  S
+ SNW                                       0                    0
+ CIG   7  4  4  3  3  3  4  4  4  4  4  3  3  4  6  6  7  7  7  7  7
+ VIS   7  5  3  2  2  3  4  5  4  5  4  4  3  5  7  7  7  7  7  7  7
+ OBV   N BR BR BR BR BR BR BR BR BR BR BR BR BR  N  N  N  N  N  N  N
+
+
+
+ + * To view other MOS products +

+ * + Go back to the Statistical Modeling Branch HomePage +


+ + + \ No newline at end of file