From 1f0d76ba9edf6d39a19584d78038d87966df6bce Mon Sep 17 00:00:00 2001 From: Tom Neuber Date: Mon, 19 Aug 2024 20:12:28 +0200 Subject: [PATCH] refactor(grafanabackuper): add restore function --- cmd/main.go | 5 +- internal/cmd/backup.go | 1 + internal/cmd/restore.go | 54 +++++++++++++++++ pkg/git/project.go | 35 +++++++++++ pkg/grafana/dashboard.go | 14 +++-- pkg/grafana/schema/dashboard.go | 2 +- pkg/grafanabackuper/client.go | 4 +- pkg/grafanabackuper/message.go | 4 +- pkg/grafanabackuper/restore.go | 103 ++++++++++++++++++++++++++++++++ 9 files changed, 214 insertions(+), 8 deletions(-) create mode 100644 internal/cmd/restore.go create mode 100644 pkg/grafanabackuper/restore.go diff --git a/cmd/main.go b/cmd/main.go index bd7579b..33d3200 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -11,7 +11,10 @@ func main() { cfg := &config.Config{} rootCmd := cmd.NewRootCommand(cfg) - rootCmd.AddCommand(cmd.NewBackupCommand(cfg)) + rootCmd.AddCommand( + cmd.NewBackupCommand(cfg), + cmd.NewRestoreCommand(cfg), + ) if err := rootCmd.Execute(); err != nil { os.Exit(1) diff --git a/internal/cmd/backup.go b/internal/cmd/backup.go index e06c9fe..fc7629c 100644 --- a/internal/cmd/backup.go +++ b/internal/cmd/backup.go @@ -10,6 +10,7 @@ import ( "github.com/spf13/cobra" ) +//nolint:dupl // This function may be the same as or similar to other cmd functions. func NewBackupCommand(c *config.Config) *cobra.Command { backupCmd := &cobra.Command{ Use: "backup", diff --git a/internal/cmd/restore.go b/internal/cmd/restore.go new file mode 100644 index 0000000..247f798 --- /dev/null +++ b/internal/cmd/restore.go @@ -0,0 +1,54 @@ +package cmd + +import ( + "context" + "os" + + "git.ar21.de/yolokube/grafana-backuper/internal/config" + "git.ar21.de/yolokube/grafana-backuper/pkg/grafanabackuper" + "github.com/rs/zerolog/log" + "github.com/spf13/cobra" +) + +//nolint:dupl // This function may be the same as or similar to other cmd functions. +func NewRestoreCommand(c *config.Config) *cobra.Command { + restoreCmd := &cobra.Command{ + Use: "restore", + Short: "Restore the dashboards from a git repository to grafana.", + Long: "Restore the dashboards from a git repository to grafana.", + Run: func(cmd *cobra.Command, _ []string) { + if err := restore(cmd.Context(), c); err != nil { + log.Fatal().Err(err).Send() + } + }, + PreRun: func(cmd *cobra.Command, _ []string) { + errs := c.Validate() + for _, err := range errs { + log.Error().Err(err).Send() + } + + if len(errs) > 0 { + if err := cmd.Help(); err != nil { + log.Error().Err(err).Send() + } + os.Exit(1) + } + }, + } + + restoreCmd.PersistentFlags().BoolVar(&c.ForceCommits, "force", false, "Force dashboards / ignore existence check") + + return restoreCmd +} + +func restore(ctx context.Context, cfg *config.Config) error { + client := grafanabackuper.NewClient( + grafanabackuper.WithZerologLogger(cfg.Logger), + ) + + if err := client.Prepare(ctx, cfg); err != nil { + return err + } + + return client.Restore.Start(ctx, cfg) +} diff --git a/pkg/git/project.go b/pkg/git/project.go index 789deb8..21d9bb7 100644 --- a/pkg/git/project.go +++ b/pkg/git/project.go @@ -4,6 +4,8 @@ import ( "context" "errors" "io" + "path/filepath" + "strings" "github.com/go-git/go-billy/v5" "github.com/go-git/go-billy/v5/memfs" @@ -214,3 +216,36 @@ func (p *Project) Push(ctx context.Context) error { return p.repository.PushContext(ctx, &pushOpts) } + +func (p *Project) ListJSONFiles(directory string) ([]string, error) { + files, err := p.fs.ReadDir(directory) + if err != nil { + return nil, err + } + + var allFiles []string + for _, file := range files { + if file.IsDir() { + var childFiles []string + childFiles, err = p.ListJSONFiles(filepath.Join(directory, file.Name())) + if err != nil { + return nil, err + } + allFiles = append(allFiles, childFiles...) + } else if strings.HasSuffix(file.Name(), ".json") { + allFiles = append(allFiles, filepath.Join(directory, file.Name())) + } + } + + return allFiles, nil +} + +func (p *Project) ReadFile(filepath string) ([]byte, error) { + file, err := p.fs.Open(filepath) + if err != nil { + return nil, err + } + defer file.Close() + + return io.ReadAll(file) +} diff --git a/pkg/grafana/dashboard.go b/pkg/grafana/dashboard.go index 08a850d..666be85 100644 --- a/pkg/grafana/dashboard.go +++ b/pkg/grafana/dashboard.go @@ -79,9 +79,6 @@ type DashboardCreateOpts struct { } func (o DashboardCreateOpts) Validate() error { - if o.FolderUID == "" && o.FolderID == 0 { - return errors.New("folder ID or UID missing") - } if o.Dashboard == nil { return errors.New("dashboard is nil") } @@ -111,7 +108,7 @@ func (c *DashboardClient) Create( reqBody.Message = opts.Message if opts.FolderUID != "" { reqBody.FolderUID = opts.FolderUID - } else { + } else if opts.FolderID > 0 { reqBody.FolderID = opts.FolderID } @@ -142,3 +139,12 @@ func (c *DashboardClient) Delete(ctx context.Context, uid string) (*Response, er return c.client.do(req, nil) } + +func (c *DashboardClient) ParseRaw(raw []byte) (*Dashboard, error) { + var body schema.Dashboard + if err := json.Unmarshal(raw, &body); err != nil { + return nil, err + } + + return DashboardFromSchema(body), nil +} diff --git a/pkg/grafana/schema/dashboard.go b/pkg/grafana/schema/dashboard.go index c381608..c6207fb 100644 --- a/pkg/grafana/schema/dashboard.go +++ b/pkg/grafana/schema/dashboard.go @@ -3,7 +3,7 @@ package schema import "time" type DashboardCreateRequest struct { - Dashboard any `json:"dasboard"` + Dashboard any `json:"dashboard"` FolderID uint `json:"folderId,omitempty"` FolderUID string `json:"folderUid"` Message string `json:"message,omitempty"` diff --git a/pkg/grafanabackuper/client.go b/pkg/grafanabackuper/client.go index 23cf7e3..5fda218 100644 --- a/pkg/grafanabackuper/client.go +++ b/pkg/grafanabackuper/client.go @@ -23,7 +23,8 @@ type Client struct { logger zerolog.Logger signer *git.SignKey - Backup BackupClient + Backup BackupClient + Restore RestoreClient } func NewClient(options ...ClientOption) *Client { @@ -34,6 +35,7 @@ func NewClient(options ...ClientOption) *Client { } client.Backup = BackupClient{client: client} + client.Restore = RestoreClient{client: client} return client } diff --git a/pkg/grafanabackuper/message.go b/pkg/grafanabackuper/message.go index e239107..66a364b 100644 --- a/pkg/grafanabackuper/message.go +++ b/pkg/grafanabackuper/message.go @@ -1,6 +1,8 @@ package grafanabackuper -import "fmt" +import ( + "fmt" +) type Message struct { ID uint diff --git a/pkg/grafanabackuper/restore.go b/pkg/grafanabackuper/restore.go new file mode 100644 index 0000000..d6576d4 --- /dev/null +++ b/pkg/grafanabackuper/restore.go @@ -0,0 +1,103 @@ +package grafanabackuper + +import ( + "context" + "fmt" + "io" + + "git.ar21.de/yolokube/grafana-backuper/internal/config" + "git.ar21.de/yolokube/grafana-backuper/pkg/grafana" +) + +type RestoreClient struct { + client *Client +} + +func (r *RestoreClient) Start(ctx context.Context, cfg *config.Config) error { + files, err := r.client.git.ListJSONFiles("./") + if err != nil { + return err + } + r.client.logger.Debug().Int("files", len(files)).Msg("Collected all files") + + for _, filename := range files { + r.client.logger.Debug().Str("file", filename).Msg("Reading data") + var data []byte + data, err = r.client.git.ReadFile(filename) + if err != nil { + return err + } + + r.client.logger.Debug().Msg("Parsing raw dashboard data") + var dashboard *grafana.Dashboard + dashboard, err = r.client.grafana.Dashboard.ParseRaw(data) + if err != nil { + return err + } + + content, ok := dashboard.Dashboard.(map[string]any) + if !ok { + continue + } + + uid := fmt.Sprint(content["uid"]) + title := fmt.Sprint(content["title"]) + + r.client.logger.Debug(). + Str("dashboard-uid", uid). + Str("title", title). + Msg("Fetching current dashboard version from Grafana") + var current *grafana.Dashboard + current, _, err = r.client.grafana.Dashboard.Get(ctx, uid) + if err != nil { + return err + } + + if current.Version == dashboard.Version && !cfg.ForceCommits { + r.client.logger.Info(). + Any("dashboard-uid", uid). + Str("folder", dashboard.FolderTitle). + Str("dashboard", title). + Msg("Dashboard already up to date") + continue + } + + r.client.logger.Debug().Str("dashboard-uid", uid).Str("title", title).Msg("Syncing dashboard with Grafana") + var createResp *grafana.DashboardCreateResponse + createResp, err = r.syncDashboard(ctx, dashboard, cfg.ForceCommits) + if err != nil { + return err + } + + r.client.logger.Info().Any("resp", createResp).Msg("Created / Updated dashboard successfully") + } + + return nil +} + +func (r *RestoreClient) syncDashboard( + ctx context.Context, + d *grafana.Dashboard, + force bool, +) (*grafana.DashboardCreateResponse, error) { + createOpts := grafana.DashboardCreateOpts{ + Dashboard: d.Dashboard, + FolderUID: d.FolderUID, + Message: "sync git repository to grafana", + Overwrite: force, + } + + r.client.logger.Debug().Msg("Validating create options") + if err := createOpts.Validate(); err != nil { + return nil, err + } + + createResp, resp, err := r.client.grafana.Dashboard.Create(ctx, createOpts) + if err != nil { + body, _ := io.ReadAll(resp.Body) + r.client.logger.Debug().Str("resp", string(body)).Msg("Got error during dashboard creation / update") + return nil, err + } + + return createResp, nil +}