From f05d2be767828368b356313c4680f05466b31872 Mon Sep 17 00:00:00 2001 From: Martin Angers Date: Tue, 10 May 2022 14:25:53 -0400 Subject: [PATCH] Produce hosts' CSV report based on requested columns (#5656) --- changes/issue-5103-export-hosts-csv-columns | 1 + docs/Using-Fleet/REST-API.md | 3 +- server/fleet/geoip.go | 13 +-- server/service/hosts.go | 90 +++++++++++++++++++-- server/service/integration_core_test.go | 39 ++++++++- 5 files changed, 128 insertions(+), 18 deletions(-) create mode 100644 changes/issue-5103-export-hosts-csv-columns diff --git a/changes/issue-5103-export-hosts-csv-columns b/changes/issue-5103-export-hosts-csv-columns new file mode 100644 index 0000000000..732affeb95 --- /dev/null +++ b/changes/issue-5103-export-hosts-csv-columns @@ -0,0 +1 @@ +* Add the `columns` query parameter to the export hosts as CSV API endpoint to select the list of columns to include. diff --git a/docs/Using-Fleet/REST-API.md b/docs/Using-Fleet/REST-API.md index f2a16b173b..5943474f79 100644 --- a/docs/Using-Fleet/REST-API.md +++ b/docs/Using-Fleet/REST-API.md @@ -2582,6 +2582,7 @@ requested by a web browser. | Name | Type | In | Description | | ----------------------- | ------- | ----- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | format | string | query | **Required**, must be "csv" (only supported format for now). | +| columns | string | query | Comma-delimited list of columns to include in the report (returns all columns if none is specified). | | order_key | string | query | What to order results by. Can be any column in the hosts table. | | order_direction | string | query | **Requires `order_key`**. The direction of the order given the order key. Options include `asc` and `desc`. Default is `asc`. | | status | string | query | Indicates the status of the hosts to return. Can either be `new`, `online`, `offline`, or `mia`. | @@ -2594,7 +2595,7 @@ requested by a web browser. #### Example -`GET /api/v1/fleet/hosts/report?software_id=123&format=csv` +`GET /api/v1/fleet/hosts/report?software_id=123&format=csv&columns=hostname,primary_ip,platform` ##### Default response diff --git a/server/fleet/geoip.go b/server/fleet/geoip.go index ef93cf9cf8..597af53944 100644 --- a/server/fleet/geoip.go +++ b/server/fleet/geoip.go @@ -3,23 +3,24 @@ package fleet import ( "context" "errors" + "net" + "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" "github.com/oschwald/geoip2-golang" - "net" ) var notCityDBError = geoip2.InvalidMethodError{} type GeoLocation struct { - CountryISO string `json:"country_iso"` - CityName string `json:"city_name"` - Geometry *Geometry `json:"geometry,omitempty"` + CountryISO string `json:"country_iso" csv:"-"` + CityName string `json:"city_name" csv:"-"` + Geometry *Geometry `json:"geometry,omitempty" csv:"-"` } type Geometry struct { - Type string `json:"type"` - Coordinates []float64 `json:"coordinates"` + Type string `json:"type" csv:"-"` + Coordinates []float64 `json:"coordinates" csv:"-"` } type GeoIP interface { diff --git a/server/service/hosts.go b/server/service/hosts.go index 1be56411f8..1fa51f80c7 100644 --- a/server/service/hosts.go +++ b/server/service/hosts.go @@ -1,9 +1,13 @@ package service import ( + "bytes" "context" + "encoding/csv" "fmt" + "io" "net/http" + "strings" "time" "github.com/fleetdm/fleet/v4/server/contexts/authz" @@ -19,10 +23,10 @@ import ( // rendering in the UI. type HostResponse struct { *fleet.Host - Status fleet.HostStatus `json:"status"` - DisplayText string `json:"display_text"` - Labels []fleet.Label `json:"labels,omitempty"` - Geolocation *fleet.GeoLocation `json:"geolocation,omitempty"` + Status fleet.HostStatus `json:"status" csv:"status"` + DisplayText string `json:"display_text" csv:"display_text"` + Labels []fleet.Label `json:"labels,omitempty" csv:"-"` + Geolocation *fleet.GeoLocation `json:"geolocation,omitempty" csv:"-"` } func hostResponseForHost(ctx context.Context, svc fleet.Service, host *fleet.Host) (*HostResponse, error) { @@ -855,20 +859,74 @@ type hostsReportRequest struct { Opts fleet.HostListOptions `url:"host_options"` LabelID *uint `query:"label_id,optional"` Format string `query:"format"` + Columns string `query:"columns,optional"` } type hostsReportResponse struct { - Hosts []*fleet.Host `json:"-"` // they get rendered explicitly, in csv - Err error `json:"error,omitempty"` + Columns []string `json:"-"` // used to control the generated csv, see the hijackRender method + Hosts []*HostResponse `json:"-"` // they get rendered explicitly, in csv + Err error `json:"error,omitempty"` } func (r hostsReportResponse) error() error { return r.Err } func (r hostsReportResponse) hijackRender(ctx context.Context, w http.ResponseWriter) { + var buf bytes.Buffer + if err := gocsv.Marshal(r.Hosts, &buf); err != nil { + logging.WithErr(ctx, err) + encodeError(ctx, ctxerr.New(ctx, "failed to generate CSV file"), w) + return + } + + returnAll := len(r.Columns) == 0 + + var outRows [][]string + if !returnAll { + // read back the CSV to filter out any unwanted columns + recs, err := csv.NewReader(&buf).ReadAll() + if err != nil { + logging.WithErr(ctx, err) + encodeError(ctx, ctxerr.New(ctx, "failed to generate CSV file"), w) + return + } + + if len(recs) > 0 { + // map the header names to their field index + hdrs := make(map[string]int, len(recs)) + for i, hdr := range recs[0] { + hdrs[hdr] = i + } + + outRows = make([][]string, len(recs)) + for i, rec := range recs { + for _, col := range r.Columns { + colIx, ok := hdrs[col] + if !ok { + // invalid column name - it would be nice to catch this in the + // endpoint before processing the results, but it would require + // duplicating the list of columns from the Host's struct tags to a + // map and keep this in sync, for what is essentially a programmer + // mistake that should be caught and corrected early. + encodeError(ctx, &badRequestError{message: fmt.Sprintf("invalid column name: %q", col)}, w) + return + } + outRows[i] = append(outRows[i], rec[colIx]) + } + } + } + } + w.Header().Add("Content-Disposition", fmt.Sprintf(`attachment; filename="Hosts %s.csv"`, time.Now().Format("2006-01-02"))) w.Header().Set("Content-Type", "text/csv") w.WriteHeader(http.StatusOK) - if err := gocsv.Marshal(r.Hosts, w); err != nil { + + var err error + if returnAll { + _, err = io.Copy(w, &buf) + } else { + err = csv.NewWriter(w).WriteAll(outRows) + } + if err != nil { logging.WithErr(ctx, err) } } @@ -908,7 +966,23 @@ func hostsReportEndpoint(ctx context.Context, request interface{}, svc fleet.Ser if err != nil { return hostsReportResponse{Err: err}, nil } - return hostsReportResponse{Hosts: hosts}, nil + + hostResps := make([]*HostResponse, len(hosts)) + for i, h := range hosts { + hr, err := hostResponseForHost(ctx, svc, h) + if err != nil { + return hostsReportResponse{Err: err}, nil + } + hostResps[i] = hr + } + rawCols := strings.Split(req.Columns, ",") + var cols []string + for _, rawCol := range rawCols { + if rawCol = strings.TrimSpace(rawCol); rawCol != "" { + cols = append(cols, rawCol) + } + } + return hostsReportResponse{Columns: cols, Hosts: hostResps}, nil } type osVersionsRequest struct { diff --git a/server/service/integration_core_test.go b/server/service/integration_core_test.go index 498de983ff..87bed34e1f 100644 --- a/server/service/integration_core_test.go +++ b/server/service/integration_core_test.go @@ -4788,10 +4788,20 @@ func (s *integrationTestSuite) TestHostsReportDownload() { require.Len(t, errs.Errors, 1) assert.Equal(t, "format", errs.Errors[0].Name) + // valid format, no column specified so all columns returned res = s.DoRaw("GET", "/api/latest/fleet/hosts/report", nil, http.StatusOK, "format", "csv") rows, err := csv.NewReader(res.Body).ReadAll() res.Body.Close() require.NoError(t, err) + require.Len(t, rows, len(hosts)+1) // all hosts + header row + require.Len(t, rows[0], 43) // total number of cols + t.Log(rows[0]) + + // valid format, some columns + res = s.DoRaw("GET", "/api/latest/fleet/hosts/report", nil, http.StatusOK, "format", "csv", "columns", "hostname") + rows, err = csv.NewReader(res.Body).ReadAll() + res.Body.Close() + require.NoError(t, err) require.Len(t, rows, len(hosts)+1) require.Contains(t, rows[0], "hostname") // first row contains headers require.Contains(t, res.Header, "Content-Disposition") @@ -4800,14 +4810,14 @@ func (s *integrationTestSuite) TestHostsReportDownload() { require.Contains(t, res.Header.Get("Content-Type"), "text/csv") // pagination does not apply to this endpoint, it returns the complete list of hosts - res = s.DoRaw("GET", "/api/latest/fleet/hosts/report", nil, http.StatusOK, "format", "csv", "page", "1", "per_page", "2") + res = s.DoRaw("GET", "/api/latest/fleet/hosts/report", nil, http.StatusOK, "format", "csv", "page", "1", "per_page", "2", "columns", "hostname") rows, err = csv.NewReader(res.Body).ReadAll() res.Body.Close() require.NoError(t, err) require.Len(t, rows, len(hosts)+1) // search criteria are applied - res = s.DoRaw("GET", "/api/latest/fleet/hosts/report", nil, http.StatusOK, "format", "csv", "query", "local0") + res = s.DoRaw("GET", "/api/latest/fleet/hosts/report", nil, http.StatusOK, "format", "csv", "query", "local0", "columns", "hostname") rows, err = csv.NewReader(res.Body).ReadAll() res.Body.Close() require.NoError(t, err) @@ -4815,12 +4825,35 @@ func (s *integrationTestSuite) TestHostsReportDownload() { require.Contains(t, rows[1], hosts[0].Hostname) // with a label id - res = s.DoRaw("GET", "/api/latest/fleet/hosts/report", nil, http.StatusOK, "format", "csv", "label_id", fmt.Sprintf("%d", customLabelID)) + res = s.DoRaw("GET", "/api/latest/fleet/hosts/report", nil, http.StatusOK, "format", "csv", "columns", "hostname", "label_id", fmt.Sprintf("%d", customLabelID)) rows, err = csv.NewReader(res.Body).ReadAll() res.Body.Close() require.NoError(t, err) require.Len(t, rows, 2) // headers + member host require.Contains(t, rows[1], hosts[2].Hostname) + + // valid format but an invalid column is provided + res = s.DoRaw("GET", "/api/latest/fleet/hosts/report", nil, http.StatusBadRequest, "format", "csv", "columns", "memory,hostname,status,nosuchcolumn") + require.NoError(t, json.NewDecoder(res.Body).Decode(&errs)) + res.Body.Close() + require.Len(t, errs.Errors, 1) + require.Contains(t, errs.Errors[0].Reason, "nosuchcolumn") + + // valid format, valid columns, order is respected, sorted + res = s.DoRaw("GET", "/api/latest/fleet/hosts/report", nil, http.StatusOK, "format", "csv", "order_key", "hostname", "order_direction", "desc", "columns", "memory,hostname,status") + rows, err = csv.NewReader(res.Body).ReadAll() + res.Body.Close() + require.NoError(t, err) + require.Len(t, rows, len(hosts)+1) + require.Equal(t, []string{"memory", "hostname", "status"}, rows[0]) // first row contains headers + require.Len(t, rows[1], 3) + // status is timing-dependent, ignore in the assertion + require.Equal(t, []string{"0", "TestIntegrations/TestHostsReportDownloadfoo.local2"}, rows[1][:2]) + require.Len(t, rows[2], 3) + require.Equal(t, []string{"0", "TestIntegrations/TestHostsReportDownloadfoo.local1"}, rows[2][:2]) + require.Len(t, rows[3], 3) + require.Equal(t, []string{"0", "TestIntegrations/TestHostsReportDownloadfoo.local0"}, rows[3][:2]) + t.Log(rows) } // this test can be deleted once the "v1" version is removed.