merge in master

Makefile

@@ -36,4 +36,4 @@ release-all: release-client release-server
 all: fmt client server
 
 clean:
-	go clean ngrok/...
+	go clean -i -r ngrok/...

docs/CHANGELOG.md

@@ -1,4 +1,28 @@
 # Changelog
+## 0.23 - 09/06/2013
+- BUGFIX: Fixed a bug which caused some important HTTP headers to be omitted from request introspection and replay
+
+## 0.22 - 09/04/2013
+- FEATURE: ngrok now tunnels websocket requests
+
+## 0.21 - 08/17/2013
+- IMPROVEMENT: The ngrok web ui can now be disabled with -webport=-1
+
+## 0.20 - 08/17/2013
+- BUGFIX: Fixed a bug where ngrok would not stop its autoupdate loop even after it should stop
+
+## 0.19 - 08/17/2013
+- BUGFIX: Fixed a bug where ngrok's would loop infinitely trying to checking for updates after the second update check
+- BUGFIX: Fixed a race condition in ngrokd's metrics logging immediately after start up
+
+## 0.18 - 08/15/2013
+- BUGFIX: Fixed a bug where ngrok would compare the Host header for virtual hosting using case-sensitive comparisons
+- BUGFIX: Fixed a bug where ngrok would not include the port number in the virtual host when not serving on port 80
+- BUGFIX: Fixed a bug where ngrok would crash when trying to replay a request
+- IMPROVEMENT: ngrok can now indicate manual updates again
+- IMPROVEMENT: ngrok can now supports update channels
+- IMPROVEMENT: ngrok can now detect some updates that will fail before downloading
+
 ## 0.17 - 07/30/2013
 - BUGFIX: Fixed an issue where ngrok's registry cache would return a URL from a different protocol
 

docs/DEVELOPMENT.md

@@ -93,7 +93,7 @@ There is a stub at _src/ngrok/main/ngrokd/ngrokd.go_ for the purposes of creatin
 
 ## ngrok - the client
 ### Code
-Code for the server lives under src/ngrok/client
+Code for the client lives under src/ngrok/client
 
 ### Entry point
 The ngrok entry point is in _src/ngrok/client/main.go_.

src/ngrok/client/cli.go

@@ -125,7 +125,7 @@ func parseArgs() *Options {
 	webport := flag.Int(
 		"webport",
 		4040,
-		"The port on which the web interface is served")
+		"The port on which the web interface is served, -1 to disable")
 
 	logto := flag.String(
 		"log",

src/ngrok/client/mvc/state.go

@@ -11,7 +11,7 @@ const (
 	UpdateNone = -1 * iota
 	UpdateInstalling
 	UpdateReady
-	UpdateError
+	UpdateAvailable
 )
 
 type ConnStatus int

src/ngrok/client/update_release.go

@@ -15,67 +15,99 @@ import (
 
 const (
 	updateEndpoint = "https://dl.ngrok.com/update"
+	checkEndpoint  = "https://dl.ngrok.com/update/check"
 )
 
-func autoUpdate(ctl mvc.Controller, token string) {
-	update := func() (updateSuccessful bool) {
-		params := make(url.Values)
-		params.Add("version", version.MajorMinor())
-		params.Add("os", runtime.GOOS)
-		params.Add("arch", runtime.GOARCH)
-
-		download := update.NewDownload()
-		downloadComplete := make(chan int)
-		ctl.Go(func() {
-			for {
-				select {
-				case progress, ok := <-download.Progress:
-					if !ok {
-						close(downloadComplete)
-						return
-					} else if progress == 100 {
-						s.update = mvc.UpdateInstalling
-						ctl.Update(s)
-						close(downloadComplete)
-						return
-					} else {
-						if progress%25 == 0 {
-							log.Info("Downloading update %d%% complete", progress)
-						}
-						s.update = mvc.UpdateStatus(progress)
-						ctl.Update(s)
-					}
+func progressWatcher(s *State, ctl *ui.Controller, progress chan int, complete chan int) {
+	for {
+		select {
+		case pct, ok := <-progress:
+			if !ok {
+				close(complete)
+				return
+			} else if pct == 100 {
+				s.update = ui.UpdateInstalling
+				ctl.Update(s)
+				close(complete)
+				return
+			} else {
+				if pct%25 == 0 {
+					log.Info("Downloading update %d%% complete", pct)
 				}
+				s.update = ui.UpdateStatus(pct)
+				ctl.Update(s)
 			}
-		})
+		}
+	}
+}
 
+func autoUpdate(s *State, ctl *ui.Controller, token string) {
+	tryAgain := true
+
+	params := make(url.Values)
+	params.Add("version", version.MajorMinor())
+	params.Add("os", runtime.GOOS)
+	params.Add("arch", runtime.GOARCH)
+	params.Add("user", token)
+
+	updateUrl := updateEndpoint + "?" + params.Encode()
+	checkUrl := checkEndpoint + "?" + params.Encode()
+
+	update := func() {
 		log.Info("Checking for update")
-		err := download.UpdateFromUrl(updateEndpoint + "?" + params.Encode())
+		available, err := update.NewDownload(checkUrl).Check()
+		if err != nil {
+			log.Error("Error while checking for update: %v", err)
+			return
+		}
+
+		if !available {
+			log.Info("No update available")
+			return
+		}
+
+		// stop trying after a single download attempt
+		// XXX: improve this so the we can:
+		// 1. safely update multiple times
+		// 2. only retry after a network connection failure
+		tryAgain = false
+
+		download := update.NewDownload(updateUrl)
+		downloadComplete := make(chan int)
+
+		go progressWatcher(s, ctl, download.Progress, downloadComplete)
+
+		log.Info("Trying to update . . .")
+		err, errRecover := download.GetAndUpdate()
 		<-downloadComplete
+
 		if err != nil {
+			// log error to console
 			log.Error("Error while updating ngrok: %v", err)
-			if download.Available {
-				s.update = mvc.UpdateError
-			} else {
-				s.update = mvc.UpdateNone
+			if errRecover != nil {
+				log.Error("Error while recovering from failed ngrok update, your binary may be missing: %v", errRecover.Error())
+				params.Add("errorRecover", errRecover.Error())
 			}
 
-			// record the error to ngrok.com's servers for debugging purposes
+			// log error to ngrok.com's servers for debugging purposes
 			params.Add("error", err.Error())
-			params.Add("user", token)
-			resp, err := http.PostForm("https://dl.ngrok.com/update/error", params)
+			resp, reportErr := http.PostForm("https://dl.ngrok.com/update/error", params)
 			if err != nil {
-				log.Error("Error while reporting update error")
+				log.Error("Error while reporting update error: %v, %v", err, reportErr)
 			}
 			resp.Body.Close()
+
+			// tell the user to update manually
+			s.update = ui.UpdateAvailable
 		} else {
-			if download.Available {
-				log.Info("Marked update ready")
-				s.update = mvc.UpdateReady
-				updateSuccessful = true
+			if !download.Available {
+				// this is the way the server tells us to update manually
+				log.Info("Server wants us to update manually")
+				s.update = ui.UpdateAvailable
 			} else {
-				log.Info("No update available at this time")
-				s.update = mvc.UpdateNone
+				// tell the user the update is ready
+				log.Info("Update ready!")
+				s.update = ui.UpdateReady
 			}
 		}
 
@@ -86,11 +118,9 @@ func autoUpdate(ctl mvc.Controller, token string) {
 	// try to update immediately and then at a set interval
 	update()
 	for _ = range time.Tick(updateCheckInterval) {
-		if update() {
-			// stop trying to update if the update function is successful
-			// XXX: improve this by trying to download versions newer than
-			// the last one we downloaded
-			return
+		if !tryAgain {
+			break
 		}
+		update()
 	}
 }

src/ngrok/client/views/term/view.go

@@ -75,7 +75,7 @@ func (v *TermView) draw() {
 		updateMsg = "ngrok is updating"
 	case mvc.UpdateReady:
 		updateMsg = "ngrok has updated: restart ngrok for the new version"
-	case mvc.UpdateError:
+	case mvc.UpdateAvailable:
 		updateMsg = "new version available at https://ngrok.com"
 	default:
 		pct := float64(updateStatus) / 100.0
@@ -109,6 +109,9 @@ func (v *TermView) draw() {
 		i++
 	}
 	webAddr := fmt.Sprintf("http://localhost:%d", v.ctl.GetWebViewPort())
+	if state.GetWebPort() == -1 {
+		webAddr = "disabled"
+	}
 	v.Printf(0, i+0, "%-30s%s", "Web Interface", webAddr)
 
 	connMeter, connTimer := state.GetConnectionMetrics()

src/ngrok/client/views/web/http.go

@@ -160,7 +160,7 @@ func (whv *WebHttpView) updateHttp() {
 				continue
 			}
 
-			rawReq, err := httputil.DumpRequest(htxn.Req.Request, true)
+			rawReq, err := httputil.DumpRequestOut(htxn.Req.Request, true)
 			if err != nil {
 				whv.Error("Failed to dump request: %v", err)
 				continue
@@ -227,12 +227,11 @@ func (whv *WebHttpView) register() {
 		r.ParseForm()
 		txnid := r.Form.Get("txnid")
 		if txn, ok := whv.idToTxn[txnid]; ok {
-			bodyBytes, err := httputil.DumpRequestOut(txn.HttpTxn.Req.Request, true)
+			reqBytes, err := base64.StdEncoding.DecodeString(txn.Req.Raw)
 			if err != nil {
 				panic(err)
 			}
-			// XXX: pull the tunnel out of the transaction's user context
-			whv.ctl.PlayRequest(txn.ConnCtx.Tunnel, bodyBytes)
+			whv.ctl.PlayRequest(txn.ConnCtx.Tunnel, reqBytes)
 			w.Write([]byte(http.StatusText(200)))
 		} else {
 			http.Error(w, http.StatusText(400), 400)

src/ngrok/proto/http.go

@@ -9,6 +9,7 @@ import (
 	"net/http/httputil"
 	"ngrok/conn"
 	"ngrok/util"
+	"sync"
 	"time"
 )
 
@@ -80,6 +81,10 @@ func (h *Http) readRequests(tee *conn.Tee, lastTxn chan *HttpTxn, connCtx interf
 			tee.Warn("Failed to extract request body: %v", err)
 		}
 
+		// golang's ReadRequest/DumpRequestOut is broken. Fix up the request so it works later
+		req.URL.Scheme = "http"
+		req.URL.Host = req.Host
+
 		txn := &HttpTxn{Start: time.Now(), ConnUserCtx: connCtx}
 		txn.Req = &HttpRequest{Request: req}
 		txn.Req.BodyBytes, txn.Req.Body, err = extractBody(req.Body)
@@ -112,5 +117,29 @@ func (h *Http) readResponses(tee *conn.Tee, lastTxn chan *HttpTxn) {
 		}
 
 		h.Txns.In() <- txn
+
+		// XXX: remove web socket shim in favor of a real websocket protocol analyzer
+		if txn.Req.Header.Get("Upgrade") == "websocket" {
+			tee.Info("Upgrading to websocket")
+			var wg sync.WaitGroup
+
+			// shim for websockets
+			// in order for websockets to work, we need to continue reading all of the
+			// the bytes in the analyzer so that the joined connections will continue
+			// sending bytes to each other
+			wg.Add(2)
+			go func() {
+				ioutil.ReadAll(tee.WriteBuffer())
+				wg.Done()
+			}()
+
+			go func() {
+				ioutil.ReadAll(tee.ReadBuffer())
+				wg.Done()
+			}()
+
+			wg.Wait()
+			break
+		}
 	}
 }

src/ngrok/server/http.go

@@ -6,6 +6,7 @@ import (
 	"net"
 	"ngrok/conn"
 	"ngrok/log"
+	"strings"
 )
 
 const (
@@ -70,7 +71,7 @@ func httpHandler(tcpConn net.Conn, proto string) {
 	}
 
 	// read out the Host header from the request
-	host := req.Host
+	host := strings.ToLower(req.Host)
 	conn.Debug("Found hostname %s in request", host)
 
 	// multiplex to find the right backend host

src/ngrok/server/metrics.go

@@ -23,8 +23,6 @@ func init() {
 	} else {
 		metrics = NewLocalMetrics(30 * time.Second)
 	}
-
-	metrics.AddLogPrefix("metrics")
 }
 
 type Metrics interface {
@@ -63,7 +61,7 @@ type LocalMetrics struct {
 
 func NewLocalMetrics(reportInterval time.Duration) *LocalMetrics {
 	metrics := LocalMetrics{
-		Logger:         log.NewPrefixLogger(),
+		Logger:         log.NewPrefixLogger("metrics"),
 		reportInterval: reportInterval,
 		windowsCounter: gometrics.NewCounter(),
 		linuxCounter:   gometrics.NewCounter(),
@@ -171,7 +169,7 @@ type KeenIoMetrics struct {
 
 func NewKeenIoMetrics() *KeenIoMetrics {
 	k := &KeenIoMetrics{
-		Logger:       log.NewPrefixLogger(),
+		Logger:       log.NewPrefixLogger("metrics"),
 		ApiKey:       os.Getenv("KEEN_API_KEY"),
 		ProjectToken: os.Getenv("KEEN_PROJECT_TOKEN"),
 		Requests:     make(chan *KeenIoRequest, 100),

src/ngrok/server/tunnel.go

@@ -66,6 +66,10 @@ func registerVhost(t *Tunnel, protocol string, servingPort int) (err error) {
 		vhost = vhost[0 : len(vhost)-len(defaultPortSuffix)]
 	}
 
+	// Canonicalize by always using lower-case
+	vhost = strings.ToLower(vhost)
+	t.url = strings.ToLower(t.url)
+
 	// Register for specific hostname
 	hostname := strings.TrimSpace(t.regMsg.Hostname)
 	if hostname != "" {
@@ -88,7 +92,7 @@ func registerVhost(t *Tunnel, protocol string, servingPort int) (err error) {
 	return
 }
 
-// Create a new tunnel from aregistration message received
+// Create a new tunnel from a registration message received
 // on a control channel
 func NewTunnel(m *msg.RegMsg, ctl *Control) (t *Tunnel, err error) {
 	t = &Tunnel{

src/ngrok/version/version.go

@@ -7,7 +7,7 @@ import (
 const (
 	Proto = "1"
 	Major = "0"
-	Minor = "17"
+	Minor = "23"
 )
 
 func MajorMinor() string {