Fix handling of no content responses that contain content-encoding header

bulk/commands.go

@@ -19,12 +19,13 @@ import (
 	"github.com/pb33f/libopenapi/datamodel/low"
 	lowbase "github.com/pb33f/libopenapi/datamodel/low/base"
 	"github.com/pb33f/libopenapi/index"
-	"github.com/rest-sh/restish/cli"
-	"github.com/rest-sh/restish/openapi"
 	"github.com/spf13/afero"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 	"gopkg.in/yaml.v3"
+
+	"github.com/rest-sh/restish/cli"
+	"github.com/rest-sh/restish/openapi"
 )
 
 var afs afero.Fs = afero.NewOsFs()
@@ -76,9 +77,14 @@ func newInterpreter(expression, schemaURL string) mexpr.Interpreter {
 		// keeping in my opinion.
 		req, _ := http.NewRequest(http.MethodGet, schemaURL, nil)
 		if resp, err := cli.MakeRequest(req); err == nil && resp.StatusCode < 300 {
-			cli.DecodeResponse(resp)
 			defer resp.Body.Close()
-			if body, err := io.ReadAll(resp.Body); err == nil {
+
+			content, err := cli.DecodeResponse(resp)
+			if err != nil {
+				panic(err)
+			}
+
+			if body, err := io.ReadAll(content); err == nil {
 
 				var rootNode yaml.Node
 				var ls lowbase.Schema

cli/api.go

@@ -250,11 +250,14 @@ func Load(entrypoint string, root *cobra.Command) (API, error) {
 		if err != nil {
 			return API{}, err
 		}
-		if err := DecodeResponse(resp); err != nil {
+		defer resp.Body.Close()
+
+		content, err := DecodeResponse(resp)
+		if err != nil {
 			return API{}, err
 		}
-		defer resp.Body.Close()
-		body, err := io.ReadAll(resp.Body)
+
+		body, err := io.ReadAll(content)
 		if err != nil {
 			return API{}, err
 		}

cli/encoding.go

@@ -35,32 +35,32 @@ func buildAcceptEncodingHeader() string {
 	return strings.Join(accept, ", ")
 }
 
-// DecodeResponse will replace the response body with a decoding reader if needed.
+// DecodeResponse will return a reader to decode the response body based on the encoding.
 // Assumes the original body will be closed outside of this function.
-func DecodeResponse(resp *http.Response) error {
+func DecodeResponse(resp *http.Response) (io.Reader, error) {
 	contentEncoding := resp.Header.Get("content-encoding")
 
+	// The net/http package handles decompressing responses in some cases.
+	// When it does, it deletes the content-encoding and content-length headers.
+	// This handles pre-decoded and non-encoded responses.
 	if contentEncoding == "" {
-		// Nothing to do!
-		return nil
+		return resp.Body, nil
 	}
 
 	encoding := encodings[contentEncoding]
 
 	if encoding == nil {
-		return fmt.Errorf("unsupported content-encoding %s", contentEncoding)
+		return nil, fmt.Errorf("unsupported content-encoding %s", contentEncoding)
 	}
 
 	LogDebug("Decoding response from %s", contentEncoding)
 
 	reader, err := encoding.Reader(resp.Body)
 	if err != nil {
-		return err
+		return nil, err
 	}
 
-	resp.Body = io.NopCloser(reader)
-
-	return nil
+	return reader, nil
 }
 
 // DeflateEncoding supports gzip-encoded response content.

cli/encoding_test.go

@@ -48,6 +48,13 @@ var encodingTests = []struct {
 }
 
 func TestEncodings(parent *testing.T) {
+	AddEncoding("deflate", &DeflateEncoding{})
+	AddEncoding("gzip", &GzipEncoding{})
+	AddEncoding("br", &BrotliEncoding{})
+	parent.Cleanup(func() {
+		encodings = nil
+	})
+
 	for _, tt := range encodingTests {
 		parent.Run(tt.name, func(t *testing.T) {
 			resp := &http.Response{
@@ -57,10 +64,10 @@ func TestEncodings(parent *testing.T) {
 				Body: io.NopCloser(bytes.NewReader(tt.data)),
 			}
 
-			err := DecodeResponse(resp)
+			content, err := DecodeResponse(resp)
 			assert.NoError(t, err)
 
-			data, err := io.ReadAll(resp.Body)
+			data, err := io.ReadAll(content)
 			assert.NoError(t, err)
 			assert.Equal(t, "hello world", string(data))
 		})

cli/request.go

@@ -491,24 +491,34 @@ func (r Response) Map() map[string]any {
 // ParseResponse takes an HTTP response and tries to parse it using the
 // registered content types. It returns a map representing the request,
 func ParseResponse(resp *http.Response) (Response, error) {
-	var parsed interface{}
+	var parsed any
 
-	// Handle content encodings
-	defer resp.Body.Close()
-	if err := DecodeResponse(resp); err != nil {
-		return Response{}, err
-	}
+	if resp.StatusCode == http.StatusNoContent {
+		if resp.ContentLength > 0 {
+			return Response{}, fmt.Errorf("server returned HTTP %d but the body is %d byte(s)", http.StatusNoContent, resp.ContentLength)
+		}
+	} else {
+		// Handle content encodings
+		defer resp.Body.Close()
+		content, err := DecodeResponse(resp)
+		if err != nil {
+			return Response{}, err
+		}
 
-	data, _ := io.ReadAll(resp.Body)
+		data, err := io.ReadAll(content)
+		if err != nil {
+			return Response{}, fmt.Errorf("failed to read response body: %w", err)
+		}
 
-	if len(data) > 0 {
-		if viper.GetBool("rsh-raw") && viper.GetString("rsh-filter") == "" {
-			// Raw mode without filtering, don't parse the response.
-			parsed = data
-		} else {
-			ct := resp.Header.Get("content-type")
-			if err := Unmarshal(ct, data, &parsed); err != nil {
+		if len(data) > 0 {
+			if viper.GetBool("rsh-raw") && viper.GetString("rsh-filter") == "" {
+				// Raw mode without filtering, don't parse the response.
 				parsed = data
+			} else {
+				ct := resp.Header.Get("content-type")
+				if err := Unmarshal(ct, data, &parsed); err != nil {
+					parsed = data
+				}
 			}
 		}
 	}

cli/request_test.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/spf13/viper"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 	"gopkg.in/h2non/gock.v1"
 )
 
@@ -54,7 +55,7 @@ func TestRequestPagination(t *testing.T) {
 	assert.Equal(t, resp.Status, http.StatusOK)
 
 	// Content length should be the sum of all combined.
-	assert.Equal(t, resp.Headers["Content-Length"], "15")
+	assert.Equal(t, "15", resp.Headers["Content-Length"])
 
 	// Response body should be a concatenation of all pages.
 	assert.Equal(t, []interface{}{1.0, 2.0, 3.0, 4.0, 5.0, 6.0}, resp.Body)
@@ -197,12 +198,61 @@ func TestRequestRetryTimeout(t *testing.T) {
 		Times(2).
 		Reply(http.StatusOK).
 		Delay(2 * time.Millisecond)
-		// Note: delay seems to have a bug where subsequent requests without the
-		// delay are still delayed... For now just have it reply twice.
+	// Note: delay seems to have a bug where subsequent requests without the
+	// delay are still delayed... For now just have it reply twice.
 
 	req, _ := http.NewRequest(http.MethodGet, "http://example.com/", nil)
 	_, err := MakeRequest(req)
 
 	assert.Error(t, err)
 	assert.ErrorContains(t, err, "timed out")
 }
+
+func TestNoContentResponsesWithEncodingHeader(t *testing.T) {
+	defer gock.Off()
+
+	testCases := []struct {
+		name    string
+		headers map[string]string
+	}{
+		{
+			name: "no encoding header",
+		},
+		{
+			name: "gzip encoding header",
+			headers: map[string]string{
+				"Content-Encoding": "gzip",
+			},
+		},
+		{
+			name: "brotli encoding header",
+			headers: map[string]string{
+				"Content-Encoding": "br",
+			},
+		},
+		{
+			name: "deflate encoding header",
+			headers: map[string]string{
+				"Content-Encoding": "deflate",
+			},
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			reset(false)
+
+			gock.New("http://example.com").
+				Get("/").
+				Reply(http.StatusNoContent).
+				SetHeaders(tc.headers)
+
+			req, err := http.NewRequest(http.MethodGet, "http://example.com/", nil)
+			require.NoError(t, err)
+
+			assert.NotPanics(t, func() {
+				MakeRequestAndFormat(req)
+			})
+		})
+	}
+}

go.mod

@@ -54,7 +54,7 @@ require (
 	github.com/fsnotify/fsnotify v1.6.0 // indirect
 	github.com/golang/protobuf v1.5.2 // indirect
 	github.com/google/btree v1.1.2 // indirect
-	github.com/gorilla/css v1.0.0 // indirect
+	github.com/gorilla/css v1.0.1 // indirect
 	github.com/gosimple/unidecode v1.0.1 // indirect
 	github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
@@ -65,7 +65,7 @@ require (
 	github.com/magiconair/properties v1.8.6 // indirect
 	github.com/mattn/go-runewidth v0.0.14 // indirect
 	github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d // indirect
-	github.com/microcosm-cc/bluemonday v1.0.21 // indirect
+	github.com/microcosm-cc/bluemonday v1.0.27 // indirect
 	github.com/miekg/pkcs11 v1.0.3-0.20190429190417-a667d056470f // indirect
 	github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
 	github.com/muesli/reflow v0.3.0 // indirect

go.sum

@@ -177,8 +177,9 @@ github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
 github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
 github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g=
-github.com/gorilla/css v1.0.0 h1:BQqNyPTi50JCFMTw/b67hByjMVXZRwGha6wxVGkeihY=
 github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c=
+github.com/gorilla/css v1.0.1 h1:ntNaBIghp6JmvWnxbZKANoLyuXTPZ4cAMlo6RyhlbO8=
+github.com/gorilla/css v1.0.1/go.mod h1:BvnYkspnSzMmwRK+b8/xgNPLiIuNZr6vbZBTPQ2A3b0=
 github.com/gosimple/slug v1.13.1 h1:bQ+kpX9Qa6tHRaK+fZR0A0M2Kd7Pa5eHPPsb1JpHD+Q=
 github.com/gosimple/slug v1.13.1/go.mod h1:UiRaFH+GEilHstLUmcBgWcI42viBN7mAb818JrYOeFQ=
 github.com/gosimple/unidecode v1.0.1 h1:hZzFTMMqSswvf0LBJZCZgThIZrpDHFXux9KeGmn6T/o=
@@ -239,8 +240,9 @@ github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh
 github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE=
 github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d h1:5PJl274Y63IEHC+7izoQE9x6ikvDFZS2mDVS3drnohI=
 github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE=
-github.com/microcosm-cc/bluemonday v1.0.21 h1:dNH3e4PSyE4vNX+KlRGHT5KrSvjeUkoNPwEORjffHJg=
 github.com/microcosm-cc/bluemonday v1.0.21/go.mod h1:ytNkv4RrDrLJ2pqlsSI46O6IVXmZOBBD4SaJyDwwTkM=
+github.com/microcosm-cc/bluemonday v1.0.27 h1:MpEUotklkwCSLeH+Qdx1VJgNqLlpY2KXwXFM08ygZfk=
+github.com/microcosm-cc/bluemonday v1.0.27/go.mod h1:jFi9vgW+H7c3V0lb6nR74Ib/DIB5OBs92Dimizgw2cA=
 github.com/miekg/pkcs11 v1.0.3-0.20190429190417-a667d056470f h1:eVB9ELsoq5ouItQBr5Tj334bhPJG/MX+m7rTchmzVUQ=
 github.com/miekg/pkcs11 v1.0.3-0.20190429190417-a667d056470f/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs=
 github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ=