From 177f378715c7cbabfe1ac502af8f7a512a10f5bf Mon Sep 17 00:00:00 2001 From: Tom Neuber Date: Mon, 26 Feb 2024 09:39:45 +0100 Subject: [PATCH 1/3] pkg/geoloc/database.go: optimize database functions & resolve nil pointer dereference --- pkg/geoloc/database.go | 24 +++++++----------------- 1 file changed, 7 insertions(+), 17 deletions(-) diff --git a/pkg/geoloc/database.go b/pkg/geoloc/database.go index a1c839d..7ac500b 100644 --- a/pkg/geoloc/database.go +++ b/pkg/geoloc/database.go @@ -8,7 +8,7 @@ import ( var database *memdb.MemDB -func CreateDatabase(ipinfos []IPInfo) error { +func CreateDatabase(ipinfos []IPInfo) (err error) { schema := &memdb.DBSchema{ Tables: map[string]*memdb.TableSchema{ "ipinfo": { @@ -24,38 +24,28 @@ func CreateDatabase(ipinfos []IPInfo) error { }, } - db, err := memdb.NewMemDB(schema) + database, err = memdb.NewMemDB(schema) if err != nil { - return err + return } - txn := db.Txn(true) + txn := database.Txn(true) defer txn.Abort() for _, ipinfo := range ipinfos { - if err := txn.Insert("ipinfo", ipinfo); err != nil { - return err + if err = txn.Insert("ipinfo", ipinfo); err != nil { + return } } txn.Commit() - - database = db - return nil + return } func SearchIPNet(ipnetnum uint) (*IPInfo, error) { txn := database.Txn(false) defer txn.Abort() - raw, err := txn.First("ipinfo", "id", ipnetnum) - if err != nil { - return nil, err - } - if raw == nil { - return nil, nil - } - var ipinfo IPInfo for { raw, err := txn.First("ipinfo", "id", ipnetnum) -- 2.45.2 From ec76d2b728fb62c7a53851badc698597562780fb Mon Sep 17 00:00:00 2001 From: Tom Neuber Date: Mon, 26 Feb 2024 09:40:34 +0100 Subject: [PATCH 2/3] Add caching to minimize lookup time (closes #12) --- api/v1/server.go | 25 +++++++++++++++---------- main.go | 3 +++ pkg/geoloc/cache.go | 29 +++++++++++++++++++++++++++++ 3 files changed, 47 insertions(+), 10 deletions(-) create mode 100644 pkg/geoloc/cache.go diff --git a/api/v1/server.go b/api/v1/server.go index 1380c97..0fcc0e3 100644 --- a/api/v1/server.go +++ b/api/v1/server.go @@ -3,6 +3,7 @@ package api_v1 import ( "context" "net/http" + "time" "git.ar21.de/yolokube/country-geo-locations/pkg/geoloc" "github.com/go-chi/chi/v5" @@ -49,16 +50,20 @@ func searchIPCtx(next http.Handler) http.Handler { render.Render(w, r, errInvalidRequest(err)) return } - for { - ipinfo, err = geoloc.SearchIPNet(ipnetnum) - if err != nil { - render.Render(w, r, errNotFound) - return - } - if ipinfo != nil { - break - } - ipnetnum = ipnetnum - 8 + + newipnet, found := geoloc.IPCache.Get(ipnetnum) + if found { + ipnetnum = newipnet + } + + ipinfo, err = geoloc.SearchIPNet(ipnetnum) + if err != nil { + render.Render(w, r, errNotFound) + return + } + + if !found { + geoloc.IPCache.Set(ipnetnum, ipinfo.IPNumFrom, 2*time.Minute) } } ctx := context.WithValue(r.Context(), keyIPInfo, ipinfo) diff --git a/main.go b/main.go index e1da3ab..2aba800 100644 --- a/main.go +++ b/main.go @@ -32,6 +32,9 @@ func main() { } fmt.Println("Import data from file successful") + geoloc.IPCache = geoloc.NewCache() + fmt.Println("Cache created") + r := chi.NewRouter() r.Use(middleware.RequestID) r.Use(middleware.Logger) diff --git a/pkg/geoloc/cache.go b/pkg/geoloc/cache.go new file mode 100644 index 0000000..6b29b9d --- /dev/null +++ b/pkg/geoloc/cache.go @@ -0,0 +1,29 @@ +package geoloc + +import ( + "sync" + "time" +) + +var IPCache *Cache + +type Cache struct { + store sync.Map +} + +func NewCache() *Cache { + return &Cache{} +} + +func (c *Cache) Set(key, value uint, ttl time.Duration) { + c.store.Store(key, value) + time.AfterFunc(ttl, func() { + c.store.Delete(key) + }) +} + +func (c *Cache) Get(key uint) (uint, bool) { + output, _ := c.store.Load(key) + value, ok := output.(uint) + return value, ok +} -- 2.45.2 From 2c62bfad2f5d8f06b542e0b1d322cd91321421f8 Mon Sep 17 00:00:00 2001 From: Tom Neuber Date: Mon, 26 Feb 2024 09:48:05 +0100 Subject: [PATCH 3/3] go.sum: go mod tidy (remove unused modules) --- go.sum | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/go.sum b/go.sum index 61dd226..de192c1 100644 --- a/go.sum +++ b/go.sum @@ -9,8 +9,6 @@ github.com/alecthomas/repr v0.1.0/go.mod h1:2kn6fqh/zIyPLmm3ugklbEi5hg5wS435eygv github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/go-chi/chi/v5 v5.0.11 h1:BnpYbFZ3T3S1WMpD79r7R5ThWX40TaFB7L31Y8xqSwA= -github.com/go-chi/chi/v5 v5.0.11/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8= github.com/go-chi/chi/v5 v5.0.12 h1:9euLV5sTrTNTRUU9POmDUvfxyj6LAABLUcEWO+JJb4s= github.com/go-chi/chi/v5 v5.0.12/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8= github.com/go-chi/render v1.0.3 h1:AsXqd2a1/INaIfUSKq3G5uA8weYx20FOsM7uSoCyyt4= @@ -42,12 +40,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/praserx/ipconv v1.2.1 h1:MWGfrF+OZ0pqIuTlNlMgvJDDbohC3h751oN1+Ov3x4k= github.com/praserx/ipconv v1.2.1/go.mod h1:DSy+AKre/e3w/npsmUDMio+OR/a2rvmMdI7rerOIgqI= -github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis= -github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= -github.com/schollz/progressbar/v3 v3.14.1 h1:VD+MJPCr4s3wdhTc7OEJ/Z3dAeBzJ7yKH/P4lC5yRTI= -github.com/schollz/progressbar/v3 v3.14.1/go.mod h1:Zc9xXneTzWXF81TGoqL71u0sBPjULtEHYtj/WVgVy8E= github.com/schollz/progressbar/v3 v3.14.2 h1:EducH6uNLIWsr560zSV1KrTeUb/wZGAHqyMFIEa99ks= github.com/schollz/progressbar/v3 v3.14.2/go.mod h1:aQAZQnhF4JGFtRJiw/eobaXpsqpVQAftEQ+hLGXaRc4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -56,14 +50,8 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= -golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/term v0.14.0/go.mod h1:TySc+nGkYR6qt8km8wUhuFRTVSMIX3XPR58y2lC8vww= -golang.org/x/term v0.16.0 h1:m+B6fahuftsE9qjo0VWp2FW0mB3MTJvR0BaMQrq0pmE= -golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY= golang.org/x/term v0.17.0 h1:mkTF7LCd6WGJNL3K1Ad7kwxNfYAW6a8a8QqtMblp/4U= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -- 2.45.2