diff --git a/go.mod b/go.mod index e652d8a..f678487 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module git.ar21.de/yolokube/country-geo-locations -go 1.22.2 +go 1.23.3 require ( github.com/alecthomas/kong v1.4.0 @@ -9,17 +9,26 @@ require ( github.com/hashicorp/go-memdb v1.3.4 github.com/levigross/grequests v0.0.0-20231203190023-9c307ef1f48d github.com/praserx/ipconv v1.2.2 + github.com/prometheus/client_golang v1.20.5 github.com/schollz/progressbar/v3 v3.17.1 ) require ( github.com/ajg/form v1.5.1 // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/google/go-querystring v1.1.0 // indirect github.com/hashicorp/go-immutable-radix v1.3.1 // indirect github.com/hashicorp/golang-lru v1.0.2 // indirect + github.com/klauspost/compress v1.17.9 // indirect github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/prometheus/client_model v0.6.1 // indirect + github.com/prometheus/common v0.55.0 // indirect + github.com/prometheus/procfs v0.15.1 // indirect github.com/rivo/uniseg v0.4.7 // indirect - golang.org/x/net v0.19.0 // indirect + golang.org/x/net v0.26.0 // indirect golang.org/x/sys v0.27.0 // indirect golang.org/x/term v0.26.0 // indirect + google.golang.org/protobuf v1.34.2 // indirect ) diff --git a/go.sum b/go.sum index e81601a..4835f93 100644 --- a/go.sum +++ b/go.sum @@ -6,6 +6,10 @@ github.com/alecthomas/kong v1.4.0 h1:UL7tzGMnnY0YRMMvJyITIRX1EpO6RbBRZDNcCevy3HA github.com/alecthomas/kong v1.4.0/go.mod h1:p2vqieVMeTAnaC83txKtXe8FLke2X07aruPWXyMPQrU= github.com/alecthomas/repr v0.4.0 h1:GhI2A8MACjfegCPVq9f1FLvIBS+DrQ2KQBFZP1iFzXc= github.com/alecthomas/repr v0.4.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chengxilo/virtualterm v1.0.4 h1:Z6IpERbRVlfB8WkOmtbHiDbBANU7cimRIof7mk9/PwM= github.com/chengxilo/virtualterm v1.0.4/go.mod h1:DyxxBZz/x1iqJjFxTFcr6/x+jSpqN0iwWCOK1q10rlY= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -14,8 +18,9 @@ github.com/go-chi/chi/v5 v5.1.0 h1:acVI1TYaD+hhedDJ3r54HyA6sExp3HfXq7QWEEY/xMw= github.com/go-chi/chi/v5 v5.1.0/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8= github.com/go-chi/render v1.0.3 h1:AsXqd2a1/INaIfUSKq3G5uA8weYx20FOsM7uSoCyyt4= github.com/go-chi/render v1.0.3/go.mod h1:/gr3hVkmYR0YlEy3LxCuVRFzEu9Ruok+gFqbIofjao0= -github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/hashicorp/go-immutable-radix v1.3.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= @@ -31,28 +36,44 @@ github.com/hashicorp/golang-lru v1.0.2 h1:dV3g9Z/unq5DpblPpw+Oqcv4dU/1omnb4Ok8iP github.com/hashicorp/golang-lru v1.0.2/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg= +github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= +github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/levigross/grequests v0.0.0-20231203190023-9c307ef1f48d h1:8fVmm2qScPn4JAF/YdTtqrPP3n58FgZ4GbKTNfaPuRs= github.com/levigross/grequests v0.0.0-20231203190023-9c307ef1f48d/go.mod h1:dFu6nuJHC3u9kCDcyGrEL7LwhK2m6Mt+alyiiIjDrRY= github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc= github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ= github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/praserx/ipconv v1.2.2 h1:oz4XXNjywgoJRAnSymUET03OwSLL7JDVjQQEtl08XV8= github.com/praserx/ipconv v1.2.2/go.mod h1:DSy+AKre/e3w/npsmUDMio+OR/a2rvmMdI7rerOIgqI= +github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y= +github.com/prometheus/client_golang v1.20.5/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= +github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= +github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= +github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G1dc= +github.com/prometheus/common v0.55.0/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8= +github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= +github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/schollz/progressbar/v3 v3.17.1 h1:bI1MTaoQO+v5kzklBjYNRQLoVpe0zbyRZNK6DFkVC5U= github.com/schollz/progressbar/v3 v3.17.1/go.mod h1:RzqpnsPQNjUyIgdglUjRLgD7sVnxN1wpmBMV+UiEbL4= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= -golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= +golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= +golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= golang.org/x/sys v0.27.0 h1:wBqf8DvsY9Y/2P8gAfPDEYNuS30J4lPHJxXSb/nJZ+s= golang.org/x/sys v0.27.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.26.0 h1:WEQa6V3Gja/BhNxg540hBip/kkaYtRg3cxg4oXSw4AU= golang.org/x/term v0.26.0/go.mod h1:Si5m1o57C5nBNQo5z1iq+XDijt21BDBDp2bK0QI8e3E= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/cache/cache.go b/internal/cache/cache.go index 72af229..fbf3dd6 100644 --- a/internal/cache/cache.go +++ b/internal/cache/cache.go @@ -16,6 +16,15 @@ func NewCache(ttl time.Duration) *Cache { } } +func (c *Cache) Count() uint { + var counter uint + c.store.Range(func(_, _ any) bool { + counter++ + return true + }) + return counter +} + func (c *Cache) Set(key, value uint) { c.store.Store(key, value) time.AfterFunc(c.ttl, func() { diff --git a/cfg/cfg.go b/internal/cmd/cli.go similarity index 72% rename from cfg/cfg.go rename to internal/cmd/cli.go index 3a04f69..ed1684d 100644 --- a/cfg/cfg.go +++ b/internal/cmd/cli.go @@ -1,4 +1,4 @@ -package cfg +package cmd import ( "errors" @@ -15,15 +15,19 @@ type AppSettings struct { DataURL string CacheTTL time.Duration ReadHeaderTimeout time.Duration + EnableExporter bool + ExporterAddress string } //nolint:lll // ignore line length type CLI struct { - ServerAddress string `name:"listen-address" env:"GEOIP_LISTEN_ADDRESS" help:"Address to use for the metrics server" default:"${default_address}"` + ServerAddress string `name:"listen-address" env:"GEOIP_LISTEN_ADDRESS" help:"Address to use for the api server" default:"${default_address}"` DataFile string `name:"data-file" env:"GEOIP_DATA_FILE" help:"path to data file" default:"${default_file_path}"` DataURL string `name:"data-url" env:"GEOIP_DATA_URL" help:"url to data file"` CacheTTL string `name:"cache-ttl" env:"GEOIP_CACHE_TTL" help:"ttl for response cache" default:"${default_cache_ttl}"` ReadHeaderTimeout string `name:"read-header-timeout" env:"GEOIP_READ_HEADER_TIMEOUT" help:"timeout for reading http header" default:"${default_read_header_timeout}"` + EnableExporter bool `name:"enable-exporter" env:"GEOIP_ENABLE_EXPORTER" help:"enable prometheus exporter" default:"${default_enable_exporter}"` + ExporterAddress string `name:"exporter-address" env:"GEOIP_EXPORTER_ADDRESS" help:"Address to use for the prometheus metrics server" default:"${default_exporter_address}"` } func (c *CLI) Parse() (*AppSettings, error) { @@ -34,8 +38,10 @@ func (c *CLI) Parse() (*AppSettings, error) { "default_file_path": "./data.csv", "default_cache_ttl": "2m", "default_read_header_timeout": "3s", + "default_enable_exporter": "false", + "default_exporter_address": ":9191", }, - kong.Name("country_geo_locations"), + kong.Name("country-geo-locations"), kong.Description("🚀 Start a simple web server for GeoIP data"), kong.UsageOnError(), ) @@ -60,5 +66,7 @@ func (c *CLI) Parse() (*AppSettings, error) { DataURL: c.DataURL, CacheTTL: cacheTTL, ReadHeaderTimeout: readHeaderTimeout, + EnableExporter: c.EnableExporter, + ExporterAddress: c.ExporterAddress, }, nil } diff --git a/internal/database/database.go b/internal/database/database.go index ed941a1..12fe882 100644 --- a/internal/database/database.go +++ b/internal/database/database.go @@ -2,7 +2,10 @@ package database import ( "errors" + "os" + "time" + "git.ar21.de/yolokube/country-geo-locations/internal/cmd" "git.ar21.de/yolokube/country-geo-locations/pkg/geoloc" "github.com/hashicorp/go-memdb" ) @@ -13,10 +16,13 @@ var ( ) type Database struct { - db *memdb.MemDB + ready bool + + config *cmd.AppSettings + db *memdb.MemDB } -func NewDatabase() (*Database, error) { +func NewDatabase(config *cmd.AppSettings) (*Database, error) { database, err := memdb.NewMemDB( &memdb.DBSchema{ Tables: map[string]*memdb.TableSchema{ @@ -38,7 +44,9 @@ func NewDatabase() (*Database, error) { } return &Database{ - db: database, + ready: false, + config: config, + db: database, }, nil } @@ -53,9 +61,14 @@ func (d *Database) Load(ipinfos []geoloc.IPInfo) error { } txn.Commit() + d.ready = true return nil } +func (d *Database) IsReady() bool { + return d.ready +} + func (d *Database) SearchIPNet(ipnetnum uint) (*geoloc.IPInfo, error) { txn := d.db.Txn(false) defer txn.Abort() @@ -87,3 +100,17 @@ func (d *Database) SearchIPNet(ipnetnum uint) (*geoloc.IPInfo, error) { return &ipinfo, nil } + +func (d *Database) Timestamp() (time.Time, error) { + file, err := os.Open(d.config.DataFile) + if err != nil { + return time.Time{}, err + } + + stats, err := file.Stat() + if err != nil { + return time.Time{}, err + } + + return stats.ModTime(), nil +} diff --git a/internal/exporter/collector.go b/internal/exporter/collector.go new file mode 100644 index 0000000..dd11c26 --- /dev/null +++ b/internal/exporter/collector.go @@ -0,0 +1,46 @@ +package exporter + +import ( + "git.ar21.de/yolokube/country-geo-locations/internal/cache" + "git.ar21.de/yolokube/country-geo-locations/internal/cmd" + "git.ar21.de/yolokube/country-geo-locations/internal/database" + "github.com/prometheus/client_golang/prometheus" +) + +type Collector struct { + config *cmd.AppSettings + cache *cache.Cache + db *database.Database + metrics *Metrics + queue *RequestDataQueue +} + +func NewCollector( + config *cmd.AppSettings, + cache *cache.Cache, + db *database.Database, + queue *RequestDataQueue, +) *Collector { + return &Collector{ + config: config, + cache: cache, + db: db, + metrics: NewMetrics(), + queue: queue, + } +} + +func (c *Collector) Describe(ch chan<- *prometheus.Desc) { + ch <- c.metrics.metricCacheTTL + ch <- c.metrics.metricCurrentlyCached + ch <- c.metrics.metricDatabaseTimestamp + ch <- c.metrics.metricDatabaseReady +} + +func (c *Collector) Collect(ch chan<- prometheus.Metric) { + c.metrics.collectCacheTTLMetric(ch, c.config.CacheTTL.Seconds()) + c.metrics.collectCurrentlyCachedMetric(ch, float64(c.cache.Count())) + c.metrics.collectDatabaseTimestampMetric(ch, c.db) + c.metrics.collectDatabaseReadyMetric(ch, c.db.IsReady()) + c.metrics.collectReqeustDataMetrics(ch, c.queue) +} diff --git a/internal/exporter/metrics.go b/internal/exporter/metrics.go new file mode 100644 index 0000000..10ed73d --- /dev/null +++ b/internal/exporter/metrics.go @@ -0,0 +1,175 @@ +package exporter + +import ( + "log" + "time" + + "git.ar21.de/yolokube/country-geo-locations/internal/database" + "github.com/prometheus/client_golang/prometheus" +) + +const ( + namespace string = "country_geo_locations" + cacheSubsystem string = "cache" + dbSubsystem string = "db" + + metricLabelCacheTTL string = "ttl" + metricLabelCurrentlyCached string = "currently_cached" + metricLabelDatabaseTimestamp string = "timestamp" + metricLabelDatabaseReady string = "ready" + metricLabelRequestsTotal string = "requests_total" + metricLabelRequestLatency string = "request_latency" +) + +type Metrics struct { + metricCacheTTL *prometheus.Desc + metricCurrentlyCached *prometheus.Desc + metricDatabaseTimestamp *prometheus.Desc + metricDatabaseReady *prometheus.Desc + metricRequestsTotal *prometheus.Desc + metricRequestLatency *prometheus.Desc +} + +func NewMetrics() *Metrics { + return &Metrics{ + metricCacheTTL: prometheus.NewDesc( + prometheus.BuildFQName( + namespace, + cacheSubsystem, + metricLabelCacheTTL, + ), + "Duration for cached requests", + nil, + nil, + ), + metricCurrentlyCached: prometheus.NewDesc( + prometheus.BuildFQName( + namespace, + cacheSubsystem, + metricLabelCurrentlyCached, + ), + "Number of cached entries", + nil, + nil, + ), + metricDatabaseTimestamp: prometheus.NewDesc( + prometheus.BuildFQName( + namespace, + dbSubsystem, + metricLabelDatabaseTimestamp, + ), + "Timestamp of the CSV file", + []string{metricLabelDatabaseTimestamp}, + nil, + ), + metricDatabaseReady: prometheus.NewDesc( + prometheus.BuildFQName( + namespace, + dbSubsystem, + metricLabelDatabaseReady, + ), + "Ready status of the database", + nil, + nil, + ), + metricRequestsTotal: prometheus.NewDesc( + prometheus.BuildFQName( + namespace, + "", + metricLabelRequestsTotal, + ), + "Counter for total requests", + nil, + nil, + ), + metricRequestLatency: prometheus.NewDesc( + prometheus.BuildFQName( + namespace, + "", + metricLabelRequestLatency, + ), + "Latency statistics for requests", + nil, + nil, + ), + } +} + +func (m *Metrics) collectCacheTTLMetric(ch chan<- prometheus.Metric, ttl float64) { + ch <- prometheus.MustNewConstMetric( + m.metricCacheTTL, + prometheus.GaugeValue, + ttl, + ) +} + +func (m *Metrics) collectCurrentlyCachedMetric(ch chan<- prometheus.Metric, count float64) { + ch <- prometheus.MustNewConstMetric( + m.metricCurrentlyCached, + prometheus.GaugeValue, + count, + ) +} + +func (m *Metrics) collectDatabaseTimestampMetric(ch chan<- prometheus.Metric, db *database.Database) { + timestamp, err := db.Timestamp() + if err == nil { + ch <- prometheus.MustNewConstMetric( + m.metricDatabaseTimestamp, + prometheus.GaugeValue, + float64(timestamp.Unix()), + timestamp.String(), + ) + } else { + log.Printf("failed to read file timestamp: %v\n", err) + } +} + +func (m *Metrics) collectDatabaseReadyMetric(ch chan<- prometheus.Metric, ready bool) { + var dbReady uint8 + if ready { + dbReady = 1 + } + + ch <- prometheus.MustNewConstMetric( + m.metricDatabaseReady, + prometheus.GaugeValue, + float64(dbReady), + ) +} + +func (m *Metrics) collectReqeustDataMetrics(ch chan<- prometheus.Metric, queue *RequestDataQueue) { + var ( + count uint64 + sum float64 + ) + buckets := make(map[float64]uint64) + bucketBounds := prometheus.DefBuckets + + data := queue.ConsumeAll() + for _, r := range data { + latency := r.Latency.Seconds() + sum += latency + count++ + + for _, bound := range bucketBounds { + if latency <= bound { + buckets[bound]++ + } + } + } + + ch <- prometheus.MustNewConstMetric( + m.metricRequestsTotal, + prometheus.CounterValue, + float64(len(data)), + ) + + ch <- prometheus.MustNewConstHistogramWithCreatedTimestamp( + m.metricRequestLatency, + count, + sum, + buckets, + time.Now(), + ) +} diff --git a/internal/exporter/middleware.go b/internal/exporter/middleware.go new file mode 100644 index 0000000..f80b23d --- /dev/null +++ b/internal/exporter/middleware.go @@ -0,0 +1,68 @@ +package exporter + +import ( + "net/http" + "sync" + "time" + + "github.com/go-chi/chi/v5/middleware" +) + +type RequestData struct { + Latency time.Duration + Request *http.Request + Start time.Time +} + +type RequestDataQueue struct { + mu sync.Mutex + data []RequestData +} + +func NewRequestDataQueue() *RequestDataQueue { + return &RequestDataQueue{ + data: []RequestData{}, + } +} + +func (q *RequestDataQueue) Add(data RequestData) { + q.mu.Lock() + defer q.mu.Unlock() + q.data = append(q.data, data) +} + +func (q *RequestDataQueue) ConsumeAll() []RequestData { + q.mu.Lock() + defer q.mu.Unlock() + data := q.data + q.data = nil + return data +} + +type Middleware struct { + queue *RequestDataQueue +} + +func NewMiddleware(queue *RequestDataQueue) func(next http.Handler) http.Handler { + m := Middleware{ + queue: queue, + } + return m.handler +} + +func (m Middleware) handler(next http.Handler) http.Handler { + fn := func(w http.ResponseWriter, r *http.Request) { + start := time.Now() + ww := middleware.NewWrapResponseWriter(w, r.ProtoMajor) + next.ServeHTTP(ww, r) + + m.queue.Add( + RequestData{ + Latency: time.Since(start), + Request: r, + Start: start, + }, + ) + } + return http.HandlerFunc(fn) +} diff --git a/main.go b/main.go index 04a83cc..5d9ac8d 100644 --- a/main.go +++ b/main.go @@ -8,18 +8,22 @@ import ( "syscall" apiv1 "git.ar21.de/yolokube/country-geo-locations/api/v1" - "git.ar21.de/yolokube/country-geo-locations/cfg" "git.ar21.de/yolokube/country-geo-locations/internal/cache" + "git.ar21.de/yolokube/country-geo-locations/internal/cmd" csvimporter "git.ar21.de/yolokube/country-geo-locations/internal/csv_importer" "git.ar21.de/yolokube/country-geo-locations/internal/database" "git.ar21.de/yolokube/country-geo-locations/internal/downloader" + "git.ar21.de/yolokube/country-geo-locations/internal/exporter" "github.com/go-chi/chi/v5" "github.com/go-chi/chi/v5/middleware" "github.com/go-chi/render" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" ) func main() { - cli := cfg.CLI{} + cli := cmd.CLI{} + queue := exporter.NewRequestDataQueue() appSettings, err := cli.Parse() if err != nil { log.Fatal(err) @@ -27,6 +31,14 @@ func main() { handleGracefulShutdown() + exporterMiddleware := exporter.NewMiddleware(queue) + r := chi.NewRouter() + r.Use(middleware.RequestID) + r.Use(middleware.Logger) + r.Use(middleware.Recoverer) + r.Use(exporterMiddleware) + r.Use(render.SetContentType(render.ContentTypeJSON)) + ctx := downloader.NewContext(appSettings.DataFile, appSettings.DataURL) if !ctx.FileExists() { if downloadErr := ctx.Download(); downloadErr != nil { @@ -35,11 +47,22 @@ func main() { log.Printf("saved file to %s\n", ctx.Filename) } - db, err := database.NewDatabase() + cache := cache.NewCache(appSettings.CacheTTL) + db, err := database.NewDatabase(appSettings) if err != nil { log.Fatal("database creation failed", err) } + if appSettings.EnableExporter { + go func() { + err = enableExporter(appSettings, cache, db, queue) + if err != nil { + log.Panic(err) + } + }() + log.Println("prometheus exporter started at", appSettings.ExporterAddress) + } + log.Println("importing data from file", appSettings.DataFile) err = csvimporter.ImportCSV(appSettings.DataFile, db) if err != nil { @@ -47,15 +70,7 @@ func main() { } log.Println("imported data from file successful") - cache := cache.NewCache(appSettings.CacheTTL) lh := apiv1.NewLocationHandler(cache, db) - - r := chi.NewRouter() - r.Use(middleware.RequestID) - r.Use(middleware.Logger) - r.Use(middleware.Recoverer) - r.Use(render.SetContentType(render.ContentTypeJSON)) - r.Mount("/api/v1", apiv1.NewRouter(lh)) server := &http.Server{ @@ -70,6 +85,23 @@ func main() { } } +func enableExporter( + settings *cmd.AppSettings, + cache *cache.Cache, + db *database.Database, + queue *exporter.RequestDataQueue, +) error { + prometheus.MustRegister(exporter.NewCollector(settings, cache, db, queue)) + + metricsServer := &http.Server{ + Addr: settings.ExporterAddress, + Handler: promhttp.Handler(), + ReadHeaderTimeout: settings.ReadHeaderTimeout, + } + + return metricsServer.ListenAndServe() +} + func handleGracefulShutdown() { var signals = make(chan os.Signal, 1)