dump the body of all requests/responses regardless of content type. show content-types and number of bytes in the body. show the same body analysis for http responses

assets/body.html

@@ -1,13 +1,17 @@
 {{ define "body" }}
+<h6>
+    {{ with len .BodyBytes }} {{ . }} bytes, {{ end }}
+    {{ with .Header.Get "Content-Type" }}{{.}}{{end}}
+</h6>
 
-{{ with $json := handleJson . }}
+{{ with $json := handleJson .BodyBytes .Header }}
     <pre><code class="json">{{ $json.Str }}</code></pre>
     {{ with $json.Err }}
         <div class="alert">{{ $json.Err }}</div>
     {{ end }}
 {{ end }}
 
-{{ with $form := handleForm . }}
+{{ with $form := handleForm .BodyBytes .Header }}
     <h6>Form Params</h6>
     <table class="table params">
     {{ range $key, $values := $form }}
@@ -21,4 +25,9 @@
     </table>
 {{ end }}
 
+
+{{ with handleOther .BodyBytes .Header }}
+<pre><code>{{ .}}</code></pre>
+{{ end }}
+
 {{ end }}

assets/page.html

@@ -53,6 +53,8 @@
             {{ range .HttpRequests.Slice }}
                 <li class="txn" txnid="{{ .Id }}">
                     <div class="row">
+
+                        <!-- Request -->
                         <div class="span6">
                             {{ with .Req }}
                                 <h3>{{ .Method }} {{ .URL.Path }}</h3>
@@ -100,13 +102,41 @@
                                 </div>
                             {{ end }}
                         </div>
+
+                        <!-- Response -->
                         <div class="span6">
                             {{ with .Resp }}
                                 <h3 class="{{ classForStatus .Status }}">{{ .Status }}</h3>
 
-                                {{ with $raw := dumpResponse . }}
-                                    <pre><code class="http">{{ $raw }}</code></pre>
-                                {{ end }}
+                                <ul class="request nav nav-pills">
+                                    <li class="active"><a target="summary" href="#">Summary</a></li>
+                                    <li><a href="#" target="headers">Headers</a></li>
+                                    <li><a href="#" target="raw">Raw</a></li>
+                                </ul>
+
+                                <div class="raw">
+                                    {{ with $raw := dumpResponse . }}
+                                        <pre><code class="http">{{ $raw }}</code></pre>
+                                    {{ end }}
+                                </div>
+
+                                <div class="headers">
+                                    {{ with .Header }}
+                                        <h6>Headers</h6>
+                                        <table class="table params">
+                                        {{ range $key, $value := .}}
+                                            <tr>
+                                                <th> {{ $key }} </th>
+                                                <td> {{ $value }} </td>
+                                            </tr>
+                                        {{ end }}
+                                        </table>
+                                    {{ end }}
+                                </div>
+
+                                <div class="summary">
+                                    {{ template "body" . }}
+                                </div>
                             {{ end }}
                         </div>
                     </div>

src/ngrok/client/views/web/http.go

@@ -5,6 +5,7 @@ import (
 	"bytes"
 	"encoding/json"
 	"html/template"
+	"io"
 	"io/ioutil"
 	"net/http"
 	"net/http/httputil"
@@ -16,16 +17,26 @@ import (
 	"strings"
 )
 
-func readBody(r *http.Request) ([]byte, error) {
+func isContentType(h http.Header, ctypes ...string) bool {
+	for _, ctype := range ctypes {
+		if strings.Contains(h.Get("Content-Type"), ctype) {
+			return true
+		}
+	}
+	return false
+}
+
+func readBody(r io.Reader) ([]byte, io.ReadCloser, error) {
 	buf := new(bytes.Buffer)
-	_, err := buf.ReadFrom(r.Body)
-	r.Body = ioutil.NopCloser(buf)
-	return buf.Bytes(), err
+	_, err := buf.ReadFrom(r)
+	return buf.Bytes(), ioutil.NopCloser(buf), err
 }
 
 type WebHttpTxn struct {
 	Id string
 	*proto.HttpTxn
+	reqBody  []byte
+	respBody []byte
 }
 
 type WebHttpView struct {
@@ -63,6 +74,7 @@ func (whv *WebHttpView) update() {
 
 			if htxn.Resp == nil {
 				whtxn := &WebHttpTxn{Id: util.RandId(), HttpTxn: htxn}
+
 				// XXX: unsafe map access from multiple go routines
 				whv.idToTxn[whtxn.Id] = whtxn
 				// XXX: use return value to delete from map so we don't leak memory
@@ -77,7 +89,7 @@ func (h *WebHttpView) register() {
 		r.ParseForm()
 		txnid := r.Form.Get("txnid")
 		if txn, ok := h.idToTxn[txnid]; ok {
-			bodyBytes, err := httputil.DumpRequestOut(txn.Req, true)
+			bodyBytes, err := httputil.DumpRequestOut(txn.Req.Request, true)
 			if err != nil {
 				panic(err)
 			}
@@ -104,53 +116,53 @@ func (h *WebHttpView) register() {
 				}
 				return ""
 			},
-			"dumpResponse": func(resp *http.Response) (interface{}, error) {
-				b, err := httputil.DumpResponse(resp, true)
+			"dumpResponse": func(resp *proto.HttpResponse) (interface{}, error) {
+				b, err := httputil.DumpResponse(resp.Response, true)
 				return string(b), err
 			},
-			"dumpRequest": func(req *http.Request) (interface{}, error) {
-				b, err := httputil.DumpRequestOut(req, true)
+			"dumpRequest": func(req *proto.HttpRequest) (interface{}, error) {
+				b, err := httputil.DumpRequestOut(req.Request, true)
 				return string(b), err
 			},
-			"handleForm": func(req *http.Request) (values interface{}, err error) {
-				if !strings.Contains(req.Header.Get("Content-Type"), "application/x-www-form-urlencoded") {
+			"handleForm": func(b []byte, h http.Header) (values interface{}, err error) {
+				if !isContentType(h, "application/x-www-form-urlencoded") {
 					return
 				}
 
-				b, err := readBody(req)
-				if err != nil {
-					return
+				if b != nil {
+					values, err = url.ParseQuery(string(b))
 				}
-
-				values, err = url.ParseQuery(string(b))
 				return
 			},
-			"handleJson": func(req *http.Request) interface{} {
-				if !strings.Contains(req.Header.Get("Content-Type"), "application/json") {
+			"handleJson": func(raw []byte, h http.Header) interface{} {
+				if !isContentType(h, "application/json") {
 					return nil
 				}
 
-				raw, err := readBody(req)
-				if err != nil {
-					panic(err)
-				}
-
+				var err error
 				pretty := new(bytes.Buffer)
-				err = json.Indent(pretty, raw, "", "    ")
+				out := raw
+				if raw != nil {
+					err = json.Indent(pretty, raw, "", "    ")
+					if err == nil {
+						out = pretty.Bytes()
+					}
+				}
 
-				retval := struct {
+				return struct {
 					Str string
 					Err error
 				}{
-					string(pretty.Bytes()),
+					string(out),
 					err,
 				}
-
-				if err != nil {
-					retval.Str = string(raw)
+			},
+			"handleOther": func(b []byte, h http.Header) interface{} {
+				if isContentType(h, "application/json", "application/x-www-form-urlencoded") {
+					return nil
 				}
 
-				return retval
+				return string(b)
 			},
 		}
 

src/ngrok/proto/http.go

@@ -1,7 +1,10 @@
 package proto
 
 import (
+	"bytes"
 	metrics "github.com/inconshreveable/go-metrics"
+	"io"
+	"io/ioutil"
 	"net/http"
 	"net/http/httputil"
 	"ngrok/conn"
@@ -9,9 +12,19 @@ import (
 	"time"
 )
 
+type HttpRequest struct {
+	*http.Request
+	BodyBytes []byte
+}
+
+type HttpResponse struct {
+	*http.Response
+	BodyBytes []byte
+}
+
 type HttpTxn struct {
-	Req      *http.Request
-	Resp     *http.Response
+	Req      *HttpRequest
+	Resp     *HttpResponse
 	Start    time.Time
 	Duration time.Duration
 }
@@ -32,6 +45,12 @@ func NewHttp() *Http {
 	}
 }
 
+func extractBody(r io.Reader) ([]byte, io.ReadCloser, error) {
+	buf := new(bytes.Buffer)
+	_, err := buf.ReadFrom(r)
+	return buf.Bytes(), ioutil.NopCloser(buf), err
+}
+
 func (h *Http) GetName() string { return "http" }
 
 func (h *Http) WrapConn(c conn.Conn) conn.Conn {
@@ -52,10 +71,17 @@ func (h *Http) readRequests(tee *conn.Tee, lastTxn chan *HttpTxn) {
 
 		// make sure we read the body of the request so that
 		// we don't block the writer 
-		_, _ = httputil.DumpRequest(req, true)
+		_, err = httputil.DumpRequest(req, true)
 
 		h.reqMeter.Mark(1)
-		txn := &HttpTxn{Req: req, Start: time.Now()}
+		if err != nil {
+			tee.Warn("Failed to extract request body: %v", err)
+		}
+
+		txn := &HttpTxn{Start: time.Now()}
+		txn.Req = &HttpRequest{Request: req}
+		txn.Req.BodyBytes, txn.Req.Body, err = extractBody(req.Body)
+
 		lastTxn <- txn
 		h.Txns.In() <- txn
 	}
@@ -65,7 +91,7 @@ func (h *Http) readResponses(tee *conn.Tee, lastTxn chan *HttpTxn) {
 	for {
 		var err error
 		txn := <-lastTxn
-		txn.Resp, err = http.ReadResponse(tee.ReadBuffer(), txn.Req)
+		resp, err := http.ReadResponse(tee.ReadBuffer(), txn.Req.Request)
 		txn.Duration = time.Since(txn.Start)
 		h.reqTimer.Update(txn.Duration)
 		if err != nil {
@@ -75,7 +101,13 @@ func (h *Http) readResponses(tee *conn.Tee, lastTxn chan *HttpTxn) {
 		}
 		// make sure we read the body of the response so that
 		// we don't block the reader 
-		_, _ = httputil.DumpResponse(txn.Resp, true)
+		_, _ = httputil.DumpResponse(resp, true)
+
+		txn.Resp = &HttpResponse{Response: resp}
+		txn.Resp.BodyBytes, txn.Resp.Body, err = extractBody(resp.Body)
+		if err != nil {
+			tee.Warn("Failed to extract response body: %v", err)
+		}
 
 		h.Txns.In() <- txn
 	}