yolokube/country-geo-locations
yolokube/country-geo-locations
2
0
Fork 0
Code Issues 1 Pull requests Projects Releases Packages 1 Wiki Activity

Optimize lookup speed / add net address cache #13

Merged
tom merged 3 commits from optimize_lookup_speed into main 2024-02-26 09:55:05 +01:00
Conversation 0 Commits 3 Files changed 5 +54 -39
5 changed files with 54 additions and 39 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

25
api/v1/server.go
View file

@ -3,6 +3,7 @@ package api_v1
import (
"context"
"net/http"
"time"
"git.ar21.de/yolokube/country-geo-locations/pkg/geoloc"
"github.com/go-chi/chi/v5"
@ -49,16 +50,20 @@ func searchIPCtx(next http.Handler) http.Handler {
render.Render(w, r, errInvalidRequest(err))
return
}
for {
ipinfo, err = geoloc.SearchIPNet(ipnetnum)
if err != nil {
render.Render(w, r, errNotFound)
return
}
if ipinfo != nil {
break
}
ipnetnum = ipnetnum - 8
newipnet, found := geoloc.IPCache.Get(ipnetnum)
if found {
ipnetnum = newipnet
}
ipinfo, err = geoloc.SearchIPNet(ipnetnum)
if err != nil {
render.Render(w, r, errNotFound)
return
}
if !found {
geoloc.IPCache.Set(ipnetnum, ipinfo.IPNumFrom, 2*time.Minute)
}
}
ctx := context.WithValue(r.Context(), keyIPInfo, ipinfo)

12
go.sum
View file

@ -9,8 +9,6 @@ github.com/alecthomas/repr v0.1.0/go.mod h1:2kn6fqh/zIyPLmm3ugklbEi5hg5wS435eygv
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/go-chi/chi/v5 v5.0.11 h1:BnpYbFZ3T3S1WMpD79r7R5ThWX40TaFB7L31Y8xqSwA=
github.com/go-chi/chi/v5 v5.0.11/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8=
github.com/go-chi/chi/v5 v5.0.12 h1:9euLV5sTrTNTRUU9POmDUvfxyj6LAABLUcEWO+JJb4s=
github.com/go-chi/chi/v5 v5.0.12/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8=
github.com/go-chi/render v1.0.3 h1:AsXqd2a1/INaIfUSKq3G5uA8weYx20FOsM7uSoCyyt4=
@ -42,12 +40,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/praserx/ipconv v1.2.1 h1:MWGfrF+OZ0pqIuTlNlMgvJDDbohC3h751oN1+Ov3x4k=
github.com/praserx/ipconv v1.2.1/go.mod h1:DSy+AKre/e3w/npsmUDMio+OR/a2rvmMdI7rerOIgqI=
github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis=
github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
github.com/schollz/progressbar/v3 v3.14.1 h1:VD+MJPCr4s3wdhTc7OEJ/Z3dAeBzJ7yKH/P4lC5yRTI=
github.com/schollz/progressbar/v3 v3.14.1/go.mod h1:Zc9xXneTzWXF81TGoqL71u0sBPjULtEHYtj/WVgVy8E=
github.com/schollz/progressbar/v3 v3.14.2 h1:EducH6uNLIWsr560zSV1KrTeUb/wZGAHqyMFIEa99ks=
github.com/schollz/progressbar/v3 v3.14.2/go.mod h1:aQAZQnhF4JGFtRJiw/eobaXpsqpVQAftEQ+hLGXaRc4=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
@ -56,14 +50,8 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV
golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c=
golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU=
golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y=
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.14.0/go.mod h1:TySc+nGkYR6qt8km8wUhuFRTVSMIX3XPR58y2lC8vww=
golang.org/x/term v0.16.0 h1:m+B6fahuftsE9qjo0VWp2FW0mB3MTJvR0BaMQrq0pmE=
golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY=
golang.org/x/term v0.17.0 h1:mkTF7LCd6WGJNL3K1Ad7kwxNfYAW6a8a8QqtMblp/4U=
golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

3
main.go
View file

@ -32,6 +32,9 @@ func main() {
}
fmt.Println("Import data from file successful")
geoloc.IPCache = geoloc.NewCache()
fmt.Println("Cache created")
r := chi.NewRouter()
r.Use(middleware.RequestID)
r.Use(middleware.Logger)

29
pkg/geoloc/cache.go Normal file
View file

@ -0,0 +1,29 @@
package geoloc
import (
"sync"
"time"
)
var IPCache *Cache
type Cache struct {
store sync.Map
}
func NewCache() *Cache {
return &Cache{}
}
func (c *Cache) Set(key, value uint, ttl time.Duration) {
c.store.Store(key, value)
time.AfterFunc(ttl, func() {
c.store.Delete(key)
})
}
func (c *Cache) Get(key uint) (uint, bool) {
output, _ := c.store.Load(key)
value, ok := output.(uint)
return value, ok
}

24
pkg/geoloc/database.go
View file

@ -8,7 +8,7 @@ import (
var database *memdb.MemDB
func CreateDatabase(ipinfos []IPInfo) error {
func CreateDatabase(ipinfos []IPInfo) (err error) {
schema := &memdb.DBSchema{
Tables: map[string]*memdb.TableSchema{
"ipinfo": {
@ -24,38 +24,28 @@ func CreateDatabase(ipinfos []IPInfo) error {
},
}
db, err := memdb.NewMemDB(schema)
database, err = memdb.NewMemDB(schema)
if err != nil {
return err
return
}
txn := db.Txn(true)
txn := database.Txn(true)
defer txn.Abort()
for _, ipinfo := range ipinfos {
if err := txn.Insert("ipinfo", ipinfo); err != nil {
return err
if err = txn.Insert("ipinfo", ipinfo); err != nil {
return
}
}
txn.Commit()
database = db
return nil
return
}
func SearchIPNet(ipnetnum uint) (*IPInfo, error) {
txn := database.Txn(false)
defer txn.Abort()
raw, err := txn.First("ipinfo", "id", ipnetnum)
if err != nil {
return nil, err
}
if raw == nil {
return nil, nil
}
var ipinfo IPInfo
for {
raw, err := txn.First("ipinfo", "id", ipnetnum)