Rework complete project
This commit is contained in:
parent
868965a072
commit
aebf7447c6
18 changed files with 2237 additions and 1980 deletions
2
.gitignore
vendored
2
.gitignore
vendored
|
@ -21,3 +21,5 @@
|
|||
# Go workspace file
|
||||
go.work
|
||||
|
||||
data.csv
|
||||
country_geo_locations
|
||||
|
|
25
Dockerfile
Normal file
25
Dockerfile
Normal file
|
@ -0,0 +1,25 @@
|
|||
FROM golang:1.21-bookworm AS build
|
||||
|
||||
# Create build workspace folder
|
||||
WORKDIR /workspace
|
||||
ADD . /workspace
|
||||
|
||||
# Install updates and build tools
|
||||
RUN apt-get update --yes && \
|
||||
apt-get install --yes build-essential
|
||||
|
||||
# Build the actual binary
|
||||
RUN make build
|
||||
|
||||
# -- -- -- -- -- --
|
||||
|
||||
# Set up image to run the tool
|
||||
FROM alpine
|
||||
|
||||
# Create main app folder to run from
|
||||
WORKDIR /app
|
||||
|
||||
# Copy built binary from build image
|
||||
COPY --from=build /workspace/country_geo_locations /app
|
||||
|
||||
ENTRYPOINT ["/app/country_geo_locations"]
|
11
Makefile
Normal file
11
Makefile
Normal file
|
@ -0,0 +1,11 @@
|
|||
# Build project
|
||||
.PHONY: build
|
||||
build:
|
||||
CGO_ENABLED=0 go build \
|
||||
-o country_geo_locations \
|
||||
main.go
|
||||
|
||||
# Build project docker container
|
||||
.PHONY: build/docker
|
||||
build/docker:
|
||||
docker build -t registry.neuber.io/country-geo-locations .
|
120
api/v1/server.go
Normal file
120
api/v1/server.go
Normal file
|
@ -0,0 +1,120 @@
|
|||
package api_v1
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
|
||||
"git.ar21.de/tom/country-geo-locations/pkg/geoloc"
|
||||
"github.com/go-chi/chi/v5"
|
||||
"github.com/go-chi/render"
|
||||
)
|
||||
|
||||
func ApiRouter() chi.Router {
|
||||
r := chi.NewRouter()
|
||||
|
||||
r.Use(render.SetContentType(render.ContentTypeJSON))
|
||||
|
||||
r.Get("/ping", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Write([]byte("pong"))
|
||||
})
|
||||
|
||||
r.Route("/location", func(r chi.Router) {
|
||||
r.Route("/{ipAddress}", func(r chi.Router) {
|
||||
r.Use(searchIPCtx)
|
||||
r.Get("/", searchIP)
|
||||
})
|
||||
})
|
||||
|
||||
return r
|
||||
}
|
||||
|
||||
func searchIP(w http.ResponseWriter, r *http.Request) {
|
||||
ipinfo := r.Context().Value(keyIPInfo).(*geoloc.IPInfo)
|
||||
|
||||
if err := render.Render(w, r, newIPInfoResponse(ipinfo)); err != nil {
|
||||
render.Render(w, r, errRender(err))
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
func searchIPCtx(next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
var ipinfo *geoloc.IPInfo
|
||||
var ipnetnum uint
|
||||
var err error
|
||||
|
||||
if ipAddress := chi.URLParam(r, "ipAddress"); ipAddress != "" {
|
||||
ipnetnum, err = geoloc.GetIPInfo(ipAddress)
|
||||
if err != nil {
|
||||
render.Render(w, r, errInvalidRequest(err))
|
||||
return
|
||||
}
|
||||
for {
|
||||
ipinfo, err = geoloc.SearchIPNet(ipnetnum)
|
||||
if err != nil {
|
||||
render.Render(w, r, errNotFound)
|
||||
return
|
||||
}
|
||||
if ipinfo != nil {
|
||||
break
|
||||
}
|
||||
ipnetnum = ipnetnum - 8
|
||||
}
|
||||
}
|
||||
ctx := context.WithValue(r.Context(), keyIPInfo, ipinfo)
|
||||
next.ServeHTTP(w, r.WithContext(ctx))
|
||||
})
|
||||
}
|
||||
|
||||
type errResponse struct {
|
||||
Err error `json:"-"`
|
||||
HTTPStatusCode int `json:"-"`
|
||||
|
||||
StatusText string `json:"status"`
|
||||
AppCode int64 `json:"code,omitempty"`
|
||||
ErrorText string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
func (e *errResponse) Render(w http.ResponseWriter, r *http.Request) error {
|
||||
render.Status(r, e.HTTPStatusCode)
|
||||
return nil
|
||||
}
|
||||
|
||||
func errInvalidRequest(err error) render.Renderer {
|
||||
return &errResponse{
|
||||
Err: err,
|
||||
HTTPStatusCode: 400,
|
||||
StatusText: "Invalid request",
|
||||
ErrorText: err.Error(),
|
||||
}
|
||||
}
|
||||
|
||||
func errRender(err error) render.Renderer {
|
||||
return &errResponse{
|
||||
Err: err,
|
||||
HTTPStatusCode: 422,
|
||||
StatusText: "Error rendering response",
|
||||
ErrorText: err.Error(),
|
||||
}
|
||||
}
|
||||
|
||||
var errNotFound = &errResponse{HTTPStatusCode: 404, StatusText: "Resource not found"}
|
||||
|
||||
type key int
|
||||
|
||||
const keyIPInfo key = iota
|
||||
|
||||
type ipInfoResponse struct {
|
||||
*geoloc.IPInfo
|
||||
|
||||
Elapsed int64 `json:"elapsed"`
|
||||
}
|
||||
|
||||
func (ir *ipInfoResponse) Render(w http.ResponseWriter, r *http.Request) error {
|
||||
ir.Elapsed = 10
|
||||
return nil
|
||||
}
|
||||
|
||||
func newIPInfoResponse(ipinfo *geoloc.IPInfo) *ipInfoResponse {
|
||||
return &ipInfoResponse{IPInfo: ipinfo}
|
||||
}
|
59
cfg/cfg.go
Normal file
59
cfg/cfg.go
Normal file
|
@ -0,0 +1,59 @@
|
|||
package cfg
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"github.com/alecthomas/kong"
|
||||
)
|
||||
|
||||
type AppSettings struct {
|
||||
ServerAddress string
|
||||
DataFile string
|
||||
DataURL string
|
||||
}
|
||||
|
||||
var cliStruct struct {
|
||||
ServerAddress string `name:"listen-address" env:"GEOIP_LISTEN_ADDRESS" help:"Address to use for the metrics server" default:"${default_address}"`
|
||||
DataFile string `name:"data-file" env:"GEOIP_DATA_FILE" help:"path to data file" default:"${default_file_path}"`
|
||||
DataURL string `name:"data-url" env:"GEOIP_DATA_URL" help:"url to data file" default:"${default_file_url}"`
|
||||
}
|
||||
|
||||
func Parse() *AppSettings {
|
||||
ctx := kong.Parse(
|
||||
&cliStruct,
|
||||
kong.Vars{
|
||||
"default_address": ":8080",
|
||||
"default_file_path": "./data.csv",
|
||||
"default_file_url": "https://data.neuber.io/data.csv",
|
||||
},
|
||||
kong.Name("country_geo_locations"),
|
||||
kong.Description("🚀 Start a simple web server for GeoIP data"),
|
||||
kong.UsageOnError(),
|
||||
)
|
||||
|
||||
validateFlags(ctx)
|
||||
settings := &AppSettings{
|
||||
ServerAddress: cliStruct.ServerAddress,
|
||||
DataFile: cliStruct.DataFile,
|
||||
DataURL: cliStruct.DataURL,
|
||||
}
|
||||
return settings
|
||||
}
|
||||
|
||||
func validateFlags(cliCtx *kong.Context) {
|
||||
var flagsValid = true
|
||||
var messages = []string{}
|
||||
if cliStruct.ServerAddress == "" {
|
||||
messages = append(messages, "error: invalid server address, must not be blank")
|
||||
flagsValid = false
|
||||
}
|
||||
if !flagsValid {
|
||||
cliCtx.PrintUsage(false)
|
||||
fmt.Println()
|
||||
for i := 0; i < len(messages); i++ {
|
||||
fmt.Println(messages[i])
|
||||
}
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
18
go.mod
18
go.mod
|
@ -1,3 +1,21 @@
|
|||
module git.ar21.de/tom/country-geo-locations
|
||||
|
||||
go 1.21.5
|
||||
|
||||
require (
|
||||
github.com/alecthomas/kong v0.8.1
|
||||
github.com/dustin/go-humanize v1.0.1
|
||||
github.com/go-chi/chi/v5 v5.0.10
|
||||
github.com/go-chi/render v1.0.3
|
||||
github.com/hashicorp/go-memdb v1.3.4
|
||||
github.com/levigross/grequests v0.0.0-20231203190023-9c307ef1f48d
|
||||
github.com/praserx/ipconv v1.2.1
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/ajg/form v1.5.1 // indirect
|
||||
github.com/google/go-querystring v1.1.0 // indirect
|
||||
github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
|
||||
github.com/hashicorp/golang-lru v1.0.2 // indirect
|
||||
golang.org/x/net v0.19.0 // indirect
|
||||
)
|
||||
|
|
38
go.sum
Normal file
38
go.sum
Normal file
|
@ -0,0 +1,38 @@
|
|||
github.com/ajg/form v1.5.1 h1:t9c7v8JUKu/XxOGBU0yjNpaMloxGEJhUkqFRq0ibGeU=
|
||||
github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY=
|
||||
github.com/alecthomas/assert/v2 v2.1.0 h1:tbredtNcQnoSd3QBhQWI7QZ3XHOVkw1Moklp2ojoH/0=
|
||||
github.com/alecthomas/assert/v2 v2.1.0/go.mod h1:b/+1DI2Q6NckYi+3mXyH3wFb8qG37K/DuK80n7WefXA=
|
||||
github.com/alecthomas/kong v0.8.1 h1:acZdn3m4lLRobeh3Zi2S2EpnXTd1mOL6U7xVml+vfkY=
|
||||
github.com/alecthomas/kong v0.8.1/go.mod h1:n1iCIO2xS46oE8ZfYCNDqdR0b0wZNrXAIAqro/2132U=
|
||||
github.com/alecthomas/repr v0.1.0 h1:ENn2e1+J3k09gyj2shc0dHr/yjaWSHRlrJ4DPMevDqE=
|
||||
github.com/alecthomas/repr v0.1.0/go.mod h1:2kn6fqh/zIyPLmm3ugklbEi5hg5wS435eygvNfaDQL8=
|
||||
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
|
||||
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
|
||||
github.com/go-chi/chi/v5 v5.0.10 h1:rLz5avzKpjqxrYwXNfmjkrYYXOyLJd37pz53UFHC6vk=
|
||||
github.com/go-chi/chi/v5 v5.0.10/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8=
|
||||
github.com/go-chi/render v1.0.3 h1:AsXqd2a1/INaIfUSKq3G5uA8weYx20FOsM7uSoCyyt4=
|
||||
github.com/go-chi/render v1.0.3/go.mod h1:/gr3hVkmYR0YlEy3LxCuVRFzEu9Ruok+gFqbIofjao0=
|
||||
github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM=
|
||||
github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8=
|
||||
github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU=
|
||||
github.com/hashicorp/go-immutable-radix v1.3.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
|
||||
github.com/hashicorp/go-immutable-radix v1.3.1 h1:DKHmCUm2hRBK510BaiZlwvpD40f8bJFeZnpfm2KLowc=
|
||||
github.com/hashicorp/go-immutable-radix v1.3.1/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
|
||||
github.com/hashicorp/go-memdb v1.3.4 h1:XSL3NR682X/cVk2IeV0d70N4DZ9ljI885xAEU8IoK3c=
|
||||
github.com/hashicorp/go-memdb v1.3.4/go.mod h1:uBTr1oQbtuMgd1SSGoR8YV27eT3sBHbYiNm53bMpgSg=
|
||||
github.com/hashicorp/go-uuid v1.0.0 h1:RS8zrF7PhGwyNPOtxSClXXj9HA8feRnJzgnI1RJCSnM=
|
||||
github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
|
||||
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
|
||||
github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
|
||||
github.com/hashicorp/golang-lru v1.0.2 h1:dV3g9Z/unq5DpblPpw+Oqcv4dU/1omnb4Ok8iPY6p1c=
|
||||
github.com/hashicorp/golang-lru v1.0.2/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
|
||||
github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM=
|
||||
github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg=
|
||||
github.com/levigross/grequests v0.0.0-20231203190023-9c307ef1f48d h1:8fVmm2qScPn4JAF/YdTtqrPP3n58FgZ4GbKTNfaPuRs=
|
||||
github.com/levigross/grequests v0.0.0-20231203190023-9c307ef1f48d/go.mod h1:dFu6nuJHC3u9kCDcyGrEL7LwhK2m6Mt+alyiiIjDrRY=
|
||||
github.com/praserx/ipconv v1.2.1 h1:MWGfrF+OZ0pqIuTlNlMgvJDDbohC3h751oN1+Ov3x4k=
|
||||
github.com/praserx/ipconv v1.2.1/go.mod h1:DSy+AKre/e3w/npsmUDMio+OR/a2rvmMdI7rerOIgqI=
|
||||
golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c=
|
||||
golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
58
main.go
Normal file
58
main.go
Normal file
|
@ -0,0 +1,58 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/signal"
|
||||
"syscall"
|
||||
|
||||
api_v1 "git.ar21.de/tom/country-geo-locations/api/v1"
|
||||
"git.ar21.de/tom/country-geo-locations/cfg"
|
||||
"git.ar21.de/tom/country-geo-locations/pkg/downloader"
|
||||
"git.ar21.de/tom/country-geo-locations/pkg/geoloc"
|
||||
"github.com/go-chi/chi/v5"
|
||||
"github.com/go-chi/chi/v5/middleware"
|
||||
"github.com/go-chi/render"
|
||||
)
|
||||
|
||||
func main() {
|
||||
appSettings := cfg.Parse()
|
||||
handleGracefulShutdown()
|
||||
if !downloader.FileExists(appSettings.DataFile) {
|
||||
downloader.DownloadFile(appSettings.DataFile, appSettings.DataURL)
|
||||
}
|
||||
|
||||
fmt.Printf("Import data from file...\r")
|
||||
err := geoloc.ImportCSV(appSettings.DataFile)
|
||||
if err != nil {
|
||||
fmt.Println("Import data from file failed")
|
||||
panic(err)
|
||||
}
|
||||
fmt.Println("Import data from file successful")
|
||||
|
||||
r := chi.NewRouter()
|
||||
r.Use(middleware.RequestID)
|
||||
r.Use(middleware.Logger)
|
||||
r.Use(middleware.Recoverer)
|
||||
r.Use(render.SetContentType(render.ContentTypeJSON))
|
||||
|
||||
r.Mount("/api/v1", api_v1.ApiRouter())
|
||||
|
||||
log.Printf("starting server at %s\n", appSettings.ServerAddress)
|
||||
http.ListenAndServe(appSettings.ServerAddress, r)
|
||||
}
|
||||
|
||||
func handleGracefulShutdown() {
|
||||
var signals = make(chan os.Signal, 1)
|
||||
|
||||
signal.Notify(signals, syscall.SIGTERM)
|
||||
signal.Notify(signals, syscall.SIGINT)
|
||||
|
||||
go func() {
|
||||
sig := <-signals
|
||||
log.Printf("caught signal: %+v", sig)
|
||||
os.Exit(0)
|
||||
}()
|
||||
}
|
198
pkg/downloader/downloader.go
Normal file
198
pkg/downloader/downloader.go
Normal file
|
@ -0,0 +1,198 @@
|
|||
package downloader
|
||||
|
||||
import (
|
||||
"crypto/md5"
|
||||
"crypto/sha1"
|
||||
"crypto/sha256"
|
||||
"errors"
|
||||
"fmt"
|
||||
"hash"
|
||||
"io"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/dustin/go-humanize"
|
||||
req "github.com/levigross/grequests"
|
||||
)
|
||||
|
||||
var (
|
||||
hashes = map[string]hash.Hash{
|
||||
"md5": md5.New(), //nolint: gosec
|
||||
"sha1": sha1.New(), //nolint: gosec
|
||||
"sha256": sha256.New(),
|
||||
}
|
||||
)
|
||||
|
||||
type Context struct {
|
||||
Filename string
|
||||
Filesize int64
|
||||
Reader io.Reader
|
||||
Chan chan *Response
|
||||
Closer []func() error
|
||||
hashWriter hash.Hash
|
||||
}
|
||||
|
||||
func NewContext(filename string, filesize int64, hw hash.Hash, closer ...func() error) *Context {
|
||||
return &Context{
|
||||
Filename: filename,
|
||||
Filesize: filesize,
|
||||
Chan: make(chan *Response),
|
||||
Closer: closer,
|
||||
hashWriter: hw,
|
||||
}
|
||||
}
|
||||
|
||||
func (ctx *Context) Checksum() string {
|
||||
return fmt.Sprintf("%x", ctx.hashWriter.Sum(nil))
|
||||
}
|
||||
|
||||
func (ctx *Context) SetReader(rdr io.Reader) {
|
||||
ctx.Reader = io.TeeReader(
|
||||
rdr,
|
||||
io.MultiWriter(
|
||||
ctx.hashWriter,
|
||||
&ProgressWriter{
|
||||
Filename: ctx.Filename,
|
||||
Filesize: ctx.Filesize,
|
||||
Channel: ctx.Chan,
|
||||
Start: time.Now(),
|
||||
},
|
||||
),
|
||||
)
|
||||
}
|
||||
|
||||
func (ctx *Context) Close() {
|
||||
for i := len(ctx.Closer) - 1; i >= 0; i-- {
|
||||
ctx.Closer[i]()
|
||||
}
|
||||
}
|
||||
|
||||
type ProgressWriter struct {
|
||||
Channel chan *Response
|
||||
Start time.Time
|
||||
Filename string
|
||||
Filesize int64
|
||||
Bytes int64
|
||||
}
|
||||
|
||||
func (pw *ProgressWriter) Write(data []byte) (int, error) {
|
||||
pw.Bytes = pw.Bytes + int64(len(data))
|
||||
var progress float64
|
||||
if pw.Filesize > 0 {
|
||||
progress = float64(pw.Bytes) / float64(pw.Filesize) * 100
|
||||
}
|
||||
pw.Channel <- &Response{
|
||||
Progress: progress,
|
||||
BytesPerSecond: float64(pw.Bytes) / time.Since(pw.Start).Seconds(),
|
||||
BytesTransfered: pw.Bytes,
|
||||
BytesTotal: pw.Filesize,
|
||||
}
|
||||
return len(data), nil
|
||||
}
|
||||
|
||||
type File struct {
|
||||
URL string
|
||||
Filename string
|
||||
Size int64
|
||||
}
|
||||
|
||||
type Response struct {
|
||||
Error error
|
||||
Progress float64
|
||||
BytesPerSecond float64
|
||||
BytesTransfered int64
|
||||
BytesTotal int64
|
||||
}
|
||||
|
||||
func handleError(err error, adds ...string) {
|
||||
msg := err.Error()
|
||||
if len(adds) != 0 {
|
||||
msg = fmt.Sprintf("%s %s", msg, strings.Join(adds, " "))
|
||||
}
|
||||
fmt.Println(msg)
|
||||
os.Exit(0)
|
||||
}
|
||||
|
||||
func printProgress(ctx *Context) {
|
||||
for {
|
||||
resp, ok := <-ctx.Chan
|
||||
if !ok {
|
||||
fmt.Println("\nSaved file to", ctx.Filename)
|
||||
for key, value := range hashes {
|
||||
fmt.Printf("%s: %x\n", key, value.Sum(nil))
|
||||
}
|
||||
break
|
||||
}
|
||||
if resp.Error != nil {
|
||||
handleError(resp.Error)
|
||||
}
|
||||
if resp.BytesTotal > 0 {
|
||||
fmt.Printf("\rDownloading %7s/%7s %.02f%%", humanize.Bytes(uint64(resp.BytesTransfered)), humanize.Bytes(uint64(resp.BytesTotal)), resp.Progress)
|
||||
} else {
|
||||
fmt.Printf("\rDownloading %7s", humanize.Bytes(uint64(resp.BytesTransfered)))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func DownloadFile(path, url string) {
|
||||
file := File{
|
||||
URL: url,
|
||||
Filename: path,
|
||||
}
|
||||
response, err := req.Head(file.URL, nil)
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "no such host") {
|
||||
handleError(errors.New("invalid url"))
|
||||
} else if strings.Contains(err.Error(), "certificate signed by unknown authority") {
|
||||
handleError(errors.New("certificate signed by unknown authority"))
|
||||
}
|
||||
handleError(err)
|
||||
}
|
||||
defer response.Close()
|
||||
if response.StatusCode == 404 {
|
||||
handleError(errors.New("invalid url"))
|
||||
}
|
||||
if response.StatusCode == 401 {
|
||||
handleError(errors.New("restriced access, credentials required"))
|
||||
}
|
||||
response.Close()
|
||||
|
||||
resp, err := req.Get(file.URL, nil)
|
||||
if err != nil {
|
||||
handleError(err)
|
||||
}
|
||||
|
||||
var filesize int64
|
||||
if resp.RawResponse.ContentLength != -1 {
|
||||
filesize = resp.RawResponse.ContentLength
|
||||
}
|
||||
ctx := NewContext(file.Filename, filesize, md5.New())
|
||||
ctx.SetReader(resp.RawResponse.Body)
|
||||
|
||||
go func(ctx *Context) {
|
||||
defer ctx.Close()
|
||||
|
||||
dstFile, err := os.Create(ctx.Filename)
|
||||
if err != nil {
|
||||
ctx.Chan <- &Response{Error: err}
|
||||
close(ctx.Chan)
|
||||
return
|
||||
}
|
||||
defer dstFile.Close()
|
||||
|
||||
writer := io.MultiWriter(dstFile, hashes["md5"], hashes["sha1"], hashes["sha256"])
|
||||
_, err = io.Copy(writer, ctx.Reader)
|
||||
if err != nil {
|
||||
ctx.Chan <- &Response{Error: err}
|
||||
}
|
||||
close(ctx.Chan)
|
||||
}(ctx)
|
||||
|
||||
printProgress(ctx)
|
||||
}
|
||||
|
||||
func FileExists(path string) bool {
|
||||
_, err := os.Stat(path)
|
||||
return !errors.Is(err, os.ErrNotExist)
|
||||
}
|
79
pkg/geoloc/csv_import.go
Normal file
79
pkg/geoloc/csv_import.go
Normal file
|
@ -0,0 +1,79 @@
|
|||
package geoloc
|
||||
|
||||
import (
|
||||
"encoding/csv"
|
||||
"os"
|
||||
"strconv"
|
||||
)
|
||||
|
||||
func parseCSV(filePath string) ([]IPInfo, error) {
|
||||
file, err := os.Open(filePath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
reader := csv.NewReader(file)
|
||||
|
||||
var data []IPInfo
|
||||
for {
|
||||
record, err := reader.Read()
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
|
||||
ipnumfrom, err := strconv.Atoi(record[0])
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
ipnumto, err := strconv.Atoi(record[1])
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if record[2] == "-" {
|
||||
record[2] = ""
|
||||
}
|
||||
if record[3] == "-" {
|
||||
record[3] = ""
|
||||
}
|
||||
if record[4] == "-" {
|
||||
record[4] = ""
|
||||
}
|
||||
if record[5] == "-" {
|
||||
record[5] = ""
|
||||
}
|
||||
latitude, err := strconv.ParseFloat(record[6], 32)
|
||||
if err != nil {
|
||||
latitude = 0
|
||||
}
|
||||
longitude, err := strconv.ParseFloat(record[7], 32)
|
||||
if err != nil {
|
||||
longitude = 0
|
||||
}
|
||||
|
||||
ipinfo := IPInfo{
|
||||
IPNumFrom: uint(ipnumfrom),
|
||||
IPNumTo: uint(ipnumto),
|
||||
Code: record[2],
|
||||
Country: record[3],
|
||||
State: record[4],
|
||||
City: record[5],
|
||||
Latitude: float32(latitude),
|
||||
Longitude: float32(longitude),
|
||||
}
|
||||
|
||||
data = append(data, ipinfo)
|
||||
}
|
||||
|
||||
return data, nil
|
||||
}
|
||||
|
||||
func ImportCSV(filePath string) error {
|
||||
ipinfos, err := parseCSV(filePath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = CreateDatabase(ipinfos)
|
||||
return err
|
||||
}
|
1969
pkg/geoloc/data.go
1969
pkg/geoloc/data.go
File diff suppressed because it is too large
Load diff
75
pkg/geoloc/database.go
Normal file
75
pkg/geoloc/database.go
Normal file
|
@ -0,0 +1,75 @@
|
|||
package geoloc
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/hashicorp/go-memdb"
|
||||
)
|
||||
|
||||
var database *memdb.MemDB
|
||||
|
||||
func CreateDatabase(ipinfos []IPInfo) error {
|
||||
schema := &memdb.DBSchema{
|
||||
Tables: map[string]*memdb.TableSchema{
|
||||
"ipinfo": {
|
||||
Name: "ipinfo",
|
||||
Indexes: map[string]*memdb.IndexSchema{
|
||||
"id": {
|
||||
Name: "id",
|
||||
Unique: true,
|
||||
Indexer: &memdb.UintFieldIndex{Field: "IPNumFrom"},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
db, err := memdb.NewMemDB(schema)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
txn := db.Txn(true)
|
||||
defer txn.Abort()
|
||||
|
||||
for _, ipinfo := range ipinfos {
|
||||
if err := txn.Insert("ipinfo", ipinfo); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
txn.Commit()
|
||||
|
||||
database = db
|
||||
return nil
|
||||
}
|
||||
|
||||
func SearchIPNet(ipnetnum uint) (*IPInfo, error) {
|
||||
txn := database.Txn(false)
|
||||
defer txn.Abort()
|
||||
|
||||
raw, err := txn.First("ipinfo", "id", ipnetnum)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if raw == nil {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
var ipinfo IPInfo
|
||||
for {
|
||||
raw, err := txn.First("ipinfo", "id", ipnetnum)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if raw != nil {
|
||||
ipinfo = raw.(IPInfo)
|
||||
break
|
||||
}
|
||||
if ipnetnum == 0 {
|
||||
return nil, fmt.Errorf("SearchIPNet: IP net not found")
|
||||
}
|
||||
ipnetnum = ipnetnum - 256
|
||||
}
|
||||
return &ipinfo, nil
|
||||
}
|
|
@ -1,10 +0,0 @@
|
|||
package geoloc
|
||||
|
||||
import "strings"
|
||||
|
||||
func GetCountry(input string) *Country {
|
||||
if c, exists := countries[strings.ToUpper(input)]; exists {
|
||||
return &c
|
||||
}
|
||||
return nil
|
||||
}
|
30
pkg/geoloc/static.go
Normal file
30
pkg/geoloc/static.go
Normal file
|
@ -0,0 +1,30 @@
|
|||
package geoloc
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"math"
|
||||
"net"
|
||||
"strings"
|
||||
|
||||
"github.com/praserx/ipconv"
|
||||
)
|
||||
|
||||
func GetCountry(input string) *IPInfo {
|
||||
if c, exists := countries[strings.ToUpper(input)]; exists {
|
||||
return &c
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func GetIPInfo(ipaddress string) (uint, error) {
|
||||
ip := net.ParseIP(ipaddress)
|
||||
if ip == nil {
|
||||
return 0, fmt.Errorf("no valid IP address")
|
||||
}
|
||||
ipnum, err := ipconv.IPv4ToInt(ip)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
ipnumfrom := uint(math.Floor(float64(ipnum)/float64(256))) * 256
|
||||
return ipnumfrom, nil
|
||||
}
|
1487
pkg/geoloc/static_data.go
Normal file
1487
pkg/geoloc/static_data.go
Normal file
File diff suppressed because it is too large
Load diff
|
@ -12,7 +12,7 @@ func TestGetCountry(t *testing.T) {
|
|||
tests := []struct {
|
||||
name string
|
||||
args args
|
||||
want *Country
|
||||
want *IPInfo
|
||||
}{
|
||||
{"GBR Test", args{input: "826"}, nil},
|
||||
{"USA upper Test", args{input: "USA"}, nil},
|
||||
|
@ -31,3 +31,23 @@ func TestGetCountry(t *testing.T) {
|
|||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetIPInfo(t *testing.T) {
|
||||
type args struct {
|
||||
input string
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
args args
|
||||
want uint
|
||||
}{
|
||||
{"IP address Test", args{input: "1.1.1.1"}, 16843008},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
if got, err := GetIPInfo(tt.args.input); err != nil || !reflect.DeepEqual(got, tt.want) {
|
||||
t.Errorf("GetIPInfo() - %v, want %v", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
12
pkg/geoloc/types.go
Normal file
12
pkg/geoloc/types.go
Normal file
|
@ -0,0 +1,12 @@
|
|||
package geoloc
|
||||
|
||||
type IPInfo struct {
|
||||
IPNumFrom uint `json:"ip_num_min"`
|
||||
IPNumTo uint `json:"ip_num_max"`
|
||||
Code string `json:"code"`
|
||||
Country string `json:"country"`
|
||||
State string `json:"state"`
|
||||
City string `json:"city"`
|
||||
Latitude float32 `json:"latitude"`
|
||||
Longitude float32 `json:"longitude"`
|
||||
}
|
|
@ -1,5 +1,9 @@
|
|||
{
|
||||
"$schema": "https://docs.renovatebot.com/renovate-schema.json",
|
||||
"extends": [
|
||||
"config:base",
|
||||
"regexManagers:dockerfileVersions"
|
||||
],
|
||||
"packageRules": [
|
||||
{
|
||||
"matchPackagePatterns": ["*"],
|
||||
|
|
Loading…
Reference in a new issue