mock so i dont keep hitting noaa

go.mod

@@ -3,8 +3,7 @@ module gitlab/tyrelsouza/gfsmos
 go 1.13
 
 require (
-	github.com/antchfx/xpath v1.1.4
+	github.com/stretchr/testify v1.5.1
-	github.com/antchfx/xquery v0.0.0-20180515051857-ad5b8c7a47b0
 	github.com/urfave/cli/v2 v2.1.1
-	golang.org/x/net v0.0.0-20200222125558-5a598a2470a0
+	gopkg.in/h2non/gock.v1 v1.0.15
 )

go.sum

@@ -1,23 +1,26 @@
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
-github.com/antchfx/xpath v1.1.4 h1:naPIpjBGeT3eX0Vw7E8iyHsY8FGt6EbGdkcd8EZCo+g=
-github.com/antchfx/xpath v1.1.4/go.mod h1:Yee4kTMuNiPYJ7nSNorELQMr1J33uOpXDMByNYhvtNk=
-github.com/antchfx/xquery v0.0.0-20180515051857-ad5b8c7a47b0 h1:JaCC8jz0zdMLk2m+qCCVLLLM/PL93p84w4pK3aJWj60=
-github.com/antchfx/xquery v0.0.0-20180515051857-ad5b8c7a47b0/go.mod h1:LzD22aAzDP8/dyiCKFp31He4m2GPjl0AFyzDtZzUu9M=
 github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY=
 github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
+github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 h1:2VTzZjLZBgl62/EtslCrtky5vbi9dd7HrQPQIx6wqiw=
+github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542/go.mod h1:Ow0tF8D4Kplbc8s8sSb3V2oUCygFHVp8gC3Dn6U4MNI=
+github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32 h1:W6apQkHrMkS0Muv8G/TipAy/FJl/rCYT0+EuS8+Z0z4=
+github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32/go.mod h1:9wM+0iRr9ahx58uYLpLIr5fm8diHn0JbqRycJi6w0Ms=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q=
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo=
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
-github.com/urfave/cli v1.22.2 h1:gsqYFH8bb9ekPA12kRo0hfjngWQjkJPlN9R0N78BoUo=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4=
+github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
 github.com/urfave/cli/v2 v2.1.1 h1:Qt8FeAtxE/vfdrLmR3rxR6JRE0RoVmbXu8+6kZtYU4k=
 github.com/urfave/cli/v2 v2.1.1/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ=
-golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
-golang.org/x/net v0.0.0-20200222125558-5a598a2470a0 h1:MsuvTghUPjX762sGLnGsxC3HM0B5r83wEtYcYR8/vRs=
-golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
-golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/h2non/gock.v1 v1.0.15 h1:SzLqcIlb/fDfg7UvukMpNcWsu7sI5tWwL+KCATZqks0=
+gopkg.in/h2non/gock.v1 v1.0.15/go.mod h1:sX4zAkdYX1TRGJ2JY156cFspQn4yRWn6p9EMdODlynE=
+gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

main.go

@@ -2,11 +2,8 @@ package main
 
 import (
 	"fmt"
-	"github.com/antchfx/xpath"
+	"gitlab/tyrelsouza/gfsmos/noaa_client"
-	"github.com/antchfx/xquery/html"
-	"io/ioutil"
 	"log"
-	"net/http"
 	"os"
 	"strings"
 
@@ -14,6 +11,7 @@ import (
 )
 
 func main() {
+
 	app := &cli.App{
 		Name:  "GFS MOS",
 		Usage: "Parse NOAA's GFS MOS for a given airport",
@@ -22,8 +20,9 @@ func main() {
 			if airport == "" {
 				return cli.NewExitError("please pass the airport's ICAO as an argument", 1)
 			}
-			pre := getDataFromPre(airport)
+			api := noaa_client.New(airport)
-			fmt.Println(pre)
+			pre := api.GetDataRaw()
+			fmt.Print(pre)
 			return nil
 		},
 	}
@@ -33,27 +32,3 @@ func main() {
 		log.Fatal(err)
 	}
 }
-
-func getDataFromPre(airport string) string {
-	url := fmt.Sprintf("https://www.nws.noaa.gov/cgi-bin/mos/getmav.pl?sta=%s", airport)
-
-	response, err := http.Get(url)
-	defer response.Body.Close()
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	bytes, err := ioutil.ReadAll(response.Body)
-	if err != nil {
-		log.Fatal(err)
-	}
-	doc, err := htmlquery.Parse(string(bytes))
-	if err != nil{
-		panic(err)
-	}
-	for _, n := range htmlquery.Find(doc, "//a/@href") {
-		fmt.Printf("%s \n", htmlquery.SelectAttr(n, "href")) // output href
-	}
-
-	return string(bytes)
-}

noaa_client/noaa_client.go

@@ -0,0 +1,41 @@
+package noaa_client
+
+import (
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"regexp"
+	"strings"
+)
+
+type API struct {
+	Airport string
+}
+
+func New(airport string) *API {
+	return &API{airport}
+}
+
+func (api *API) GetBody(url string) ([]byte, error) {
+	resp, err := http.Get(url)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+	body, err := ioutil.ReadAll(resp.Body)
+	// handling error and doing stuff with body that needs to be unit tested
+	return body, err
+}
+
+func (api *API) GetDataRaw() string {
+	url := fmt.Sprintf("https://www.nws.noaa.gov/cgi-bin/mos/getmav.pl?sta=%s", api.Airport)
+	body, _ := api.GetBody(url)
+
+	re := regexp.MustCompile(`(?s)<PRE.*?>(.*)</PRE>`)
+
+	for _, element := range re.FindAllStringSubmatch(string(body), -1) {
+		return strings.TrimSpace(element[1])
+	}
+
+	return ""
+}

noaa_client/noaa_client_test.go

@@ -0,0 +1,36 @@
+package noaa_client
+
+import (
+	"github.com/stretchr/testify/assert"
+	"gopkg.in/h2non/gock.v1"
+	"io/ioutil"
+	"log"
+	"testing"
+)
+func fakeResponse() string {
+	b, err := ioutil.ReadFile("../test/KeeneResponse.html") // b has type []byte
+	if err != nil {
+		log.Fatal(err)
+	}
+	return string(b)
+}
+
+
+func TestFoo(t *testing.T) {
+	defer gock.Off() // Flush pending mocks after test execution
+
+	gock.New("https://www.nws.noaa.gov").
+		Get("/cgi-bin/mos/getmav.pl").
+		MatchParam("sta", "KEEN").
+		Reply(200).
+		BodyString(fakeResponse())
+
+		// Use Client & URL from our local test server
+	api := API{
+		"KEEN",
+	}
+
+	body := api.GetDataRaw()
+
+	assert.Equal(t, "bob", body)
+}

test/KeeneResponse.html

@@ -0,0 +1,39 @@
+
+<HTML><HEAD><TITLE>GFS MOS FORECASTS</TITLE></HEAD><BODY BGCOLOR="#99CCFF">
+<p>GFS MOS FORECASTS</p>
+<HR WIDTH=75%>
+<PRE>
+ KEEN   GFS MOS GUIDANCE    2/25/2020  1800 UTC
+ DT /FEB  26                  /FEB  27                /FEB  28
+ HR   00 03 06 09 12 15 18 21 00 03 06 09 12 15 18 21 00 03 06 12 18
+ N/X              34          43          38          47       20
+ TMP  42 39 37 37 36 38 41 41 40 40 41 41 41 47 45 41 37 31 26 23 33
+ DPT  33 34 33 32 31 32 34 35 33 33 34 34 34 35 29 22 19 13 11 10  6
+ CLD  OV OV OV OV OV OV OV OV OV OV OV OV OV OV OV OV BK BK BK BK OV
+ WDR  02 01 01 01 02 04 02 04 04 07 04 04 06 20 23 23 22 22 22 24 25
+ WSP  01 01 01 01 01 01 01 03 02 06 08 18 16 19 14 10 09 11 12 14 17
+ P06        43    53    59    38    36    92    87    21    14 11 13
+ P12              60          59          98          93       19
+ Q06         0     1     1     0     0     5     4     0     0  0  0
+ Q12               1           1           4           4        0
+ T06      0/ 0  0/ 0  0/ 0  0/ 0  0/ 0  0/ 7  1/ 6  1/11  0/ 9  0/ 3
+ T12                  0/ 0        0/ 0        1/ 8        1/16  0/ 5
+ POZ   0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  1  1  3  1
+ POS   0  9  7  5  7  7  7  6  5  3  1  4  3 16 37 74 91 98 98 98 99
+ TYP   R  R  R  R  R  R  R  R  R  R  R  R  R  R  R  S  S  S  S  S  S
+ SNW                                       0                    0
+ CIG   7  4  4  3  3  3  4  4  4  4  4  3  3  4  6  6  7  7  7  7  7
+ VIS   7  5  3  2  2  3  4  5  4  5  4  4  3  5  7  7  7  7  7  7  7
+ OBV   N BR BR BR BR BR BR BR BR BR BR BR BR BR  N  N  N  N  N  N  N
+
+</PRE>
+<HR WIDTH=75%>
+<A HREF="http://www.nws.noaa.gov/mdl/synop/products.php">
+    <IMG ALT="*" ALIGN=MIDDLE BORDER="0" SRC="http://www.nws.noaa.gov/mdl/lamp/web_images/backarrow.gif"> To view other MOS products</A>
+<BR><P><P>
+    <A HREF="http://www.nws.noaa.gov/mdl/synop"> <IMG ALT="*" ALIGN=MIDDLE SRC="http://www.nws.noaa.gov/mdl/lamp/web_images/button.home.gif" BORDER="0">
+        Go back to the Statistical Modeling Branch HomePage</A>
+<P><HR><BR>
+
+</body>
+</html>