geoip: resolve node countries locally for the public views (Q7)

country/country_auto in buildProbeNodeView now come from a local
MMDB lookup of the node's public address (held v4, held v6, seen
v4, seen v6) instead of the pin alone / a hardcoded empty string.
Layered data: local file in the data folder wins, embedded factory
snapshot is the fallback, no database means the pre-flag empty
shape. A weekly job conditionally downloads a fresher copy,
validates it, atomically replaces the local file and hot-swaps the
reader; no URL configured (geoipCountryDefaultURL / the
geoip_country_url setting) means the updater stays silent. No
address ever leaves the master.
This commit is contained in:
Muse
2026-10-23 16:42:28 +00:00
parent e7f4bfed24
commit c94136c8a1
6 changed files with 423 additions and 5 deletions
+1
View File
@@ -68,6 +68,7 @@ require (
github.com/mattn/go-isatty v0.0.24 // indirect github.com/mattn/go-isatty v0.0.24 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/oschwald/maxminddb-golang v1.13.1 // indirect
github.com/pelletier/go-toml/v2 v2.4.3 // indirect github.com/pelletier/go-toml/v2 v2.4.3 // indirect
github.com/power-devops/perfstat v0.0.0-20260805114148-88456608a4f6 // indirect github.com/power-devops/perfstat v0.0.0-20260805114148-88456608a4f6 // indirect
github.com/quic-go/qpack v0.6.0 // indirect github.com/quic-go/qpack v0.6.0 // indirect
+2
View File
@@ -108,6 +108,8 @@ github.com/nicksnyder/go-i18n/v2 v2.6.1 h1:JDEJraFsQE17Dut9HFDHzCoAWGEQJom5s0TRd
github.com/nicksnyder/go-i18n/v2 v2.6.1/go.mod h1:Vee0/9RD3Quc/NmwEjzzD7VTZ+Ir7QbXocrkhOzmUKA= github.com/nicksnyder/go-i18n/v2 v2.6.1/go.mod h1:Vee0/9RD3Quc/NmwEjzzD7VTZ+Ir7QbXocrkhOzmUKA=
github.com/op/go-logging v0.0.0-20160315200505-970db520ece7 h1:lDH9UUVJtmYCjyT0CI4q8xvlXPxeZ0gYCVvWbmPlp88= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7 h1:lDH9UUVJtmYCjyT0CI4q8xvlXPxeZ0gYCVvWbmPlp88=
github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk=
github.com/oschwald/maxminddb-golang v1.13.1 h1:G3wwjdN9JmIK2o/ermkHM+98oX5fS+k5MbwsmL4MRQE=
github.com/oschwald/maxminddb-golang v1.13.1/go.mod h1:K4pgV9N/GcK694KSTmVSDTODk4IsCNThNdTmnaBZ/F8=
github.com/pelletier/go-toml/v2 v2.4.3 h1:GTRvJQutkOSftxIFD5xw9aepkYNuPWmVJpffdDPYVpY= github.com/pelletier/go-toml/v2 v2.4.3 h1:GTRvJQutkOSftxIFD5xw9aepkYNuPWmVJpffdDPYVpY=
github.com/pelletier/go-toml/v2 v2.4.3/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= github.com/pelletier/go-toml/v2 v2.4.3/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+23
View File
@@ -0,0 +1,23 @@
package job
import (
"github.com/mhsanaei/3x-ui/v3/internal/web/service"
)
// GeoIPCountryJob is the weekly refresh of the local country
// database behind the public pages' node flags (Q7: a local MMDB on
// the master, never an external per-node lookup). The service does
// the conditional download, validation and hot swap; with no update
// URL configured the pass is a no-op, so scheduling it is harmless
// on installs that never opted in.
type GeoIPCountryJob struct{}
// NewGeoIPCountryJob creates a geoip country job instance.
func NewGeoIPCountryJob() *GeoIPCountryJob {
return new(GeoIPCountryJob)
}
// Run executes one update pass.
func (j *GeoIPCountryJob) Run() {
service.UpdateGeoIPCountry()
}
+374
View File
@@ -0,0 +1,374 @@
package service
import (
_ "embed"
"fmt"
"io"
"net"
"net/http"
"os"
"path/filepath"
"strings"
"sync"
"time"
"github.com/oschwald/maxminddb-golang"
"github.com/mhsanaei/3x-ui/v3/internal/config"
"github.com/mhsanaei/3x-ui/v3/internal/database/model"
"github.com/mhsanaei/3x-ui/v3/internal/logger"
)
// Node country flags (item 7 Q7, resolved 2026-10-24): the public
// views carry an ISO country code per node, which the themes render
// as a flag. The upstream hub resolves it by calling ipinfo.io per
// node; this build never sends a node address to a third party — the
// lookup is a local MMDB read on the master, from the node's own
// reported addresses. The agent is untouched: it already reports its
// interface addresses in its facts, and the master already observes
// the connection's source addresses (SeenIPv4/SeenIPv6).
//
// The data is the DB-IP Lite country database (CC BY 4.0 by DB-IP,
// https://db-ip.com; attribution and snapshot provenance live in
// geoipdata/NOTICE), in the MMDB conversion published by the
// sapics/ip-location-db project. Three layers, in winning order:
//
// 1. a local file in the data folder (geoip-country.mmdb beside the
// panel database) — written by the weekly updater below, or by
// the operator by hand;
// 2. the factory snapshot embedded in the binary (geoipdata/), so a
// fresh install shows flags with no network at all;
// 3. no database: lookups answer "", the views render exactly the
// pre-flag shape.
//
// Reads are mmap-backed (maxminddb.Open) or run against the embedded
// bytes in place; either way a lookup is a pure in-memory tree walk
// on the snapshot path, and answers are cached per address for the
// life of the loaded database.
//go:embed geoipdata/country.mmdb
var geoipCountryEmbedded []byte
// geoipCountryFileName is the local copy's name inside the data
// folder (config.GetDBFolderPath, where x-ui.db and probe.db live).
const geoipCountryFileName = "geoip-country.mmdb"
// geoipCountryDefaultURL is the weekly updater's fallback source,
// used when the operator has not set the geoip_country_url setting.
//
// RELEASE-TIME (main session): point this at the self-hosted Gitea
// mirror asset for geoip-country.mmdb. Empty disables auto-update;
// the embedded snapshot and any hand-placed local file still apply.
const geoipCountryDefaultURL = ""
// Settings keys (main settings KV, alongside the other probe_* keys).
const (
geoipCountryURLSetting = "geoip_country_url"
geoipCountryETagSetting = "geoip_country_etag"
)
// geoipCountryMaxBytes caps one update download; the country
// database is ~8 MB today, the cap only stops a runaway response.
const geoipCountryMaxBytes = 64 << 20
// geoipCountryProbeIP / geoipCountryProbeIPv6 are the well-known
// addresses a candidate database must answer to be trusted (Google
// DNS v4, AliDNS v6 — both resolve in the DB-IP data).
const (
geoipCountryProbeIP = "8.8.8.8"
geoipCountryProbeIPv6 = "2400:3200::1"
)
// GeoIPCountryLocalPath is where the local country database lives.
func GeoIPCountryLocalPath() string {
return filepath.Join(config.GetDBFolderPath(), geoipCountryFileName)
}
// geoipCountryUpdateURL resolves the updater's source: the operator
// setting wins, else the compiled-in default (possibly empty).
func geoipCountryUpdateURL() string {
if v, ok := probeSetting(geoipCountryURLSetting); ok {
if v = strings.TrimSpace(v); v != "" {
return v
}
}
return geoipCountryDefaultURL
}
// ---- the loaded database ----
// countryRecord is the DB-IP Lite record shape: a flat country_code.
type countryRecord struct {
CountryCode string `maxminddb:"country_code"`
}
// geoipDB is one loaded database plus its per-address answer cache
// ("" answers are cached too — an unlisted address stays unlisted
// until the database is swapped).
type geoipDB struct {
reader *maxminddb.Reader
origin string // "local" | "embedded"
mu sync.Mutex
cache map[string]string
}
func (db *geoipDB) lookup(ipStr string) string {
db.mu.Lock()
if cc, ok := db.cache[ipStr]; ok {
db.mu.Unlock()
return cc
}
db.mu.Unlock()
cc := ""
var rec countryRecord
if err := db.reader.Lookup(net.ParseIP(ipStr), &rec); err == nil {
cc = rec.CountryCode
}
db.mu.Lock()
db.cache[ipStr] = cc
db.mu.Unlock()
return cc
}
var (
geoipMu sync.RWMutex
geoipCur *geoipDB
geoipOnce sync.Once
)
// currentGeoIP returns the live database, loading it on first use.
func currentGeoIP() *geoipDB {
geoipOnce.Do(func() {
if db := loadGeoIP(); db != nil {
swapGeoIP(db)
}
})
geoipMu.RLock()
defer geoipMu.RUnlock()
return geoipCur
}
// swapGeoIP makes db the live database. The replaced reader is
// deliberately NOT closed: it is mmap-backed (or backed by the
// binary's own image), a lookup racing the swap could still be
// walking it, and an mmap cannot be safely unmapped under a reader.
// The mapping is one file per week at most and is reclaimed when the
// process exits.
func swapGeoIP(db *geoipDB) {
geoipMu.Lock()
geoipCur = db
geoipMu.Unlock()
}
// loadGeoIP applies the layer order: a valid local file wins, else
// the embedded snapshot. A local file that will not open or will not
// answer falls through to the snapshot with one warning — a corrupt
// download must never take the flags (or the panel) down.
func loadGeoIP() *geoipDB {
if path := GeoIPCountryLocalPath(); path != "" {
if r, err := openGeoIPFile(path); err == nil {
return &geoipDB{reader: r, origin: "local", cache: map[string]string{}}
} else {
logger.Warningf("geoip: local country database %s unusable (%v); using the embedded snapshot", path, err)
}
}
if r, err := maxminddb.FromBytes(geoipCountryEmbedded); err == nil {
if err := probeGeoIPReader(r); err == nil {
return &geoipDB{reader: r, origin: "embedded", cache: map[string]string{}}
} else {
logger.Warningf("geoip: embedded country snapshot fails validation: %v", err)
}
} else {
logger.Warningf("geoip: embedded country snapshot will not open: %v", err)
}
return &geoipDB{origin: "none", cache: map[string]string{}}
}
// openGeoIPFile opens a database file and checks it answers.
func openGeoIPFile(path string) (*maxminddb.Reader, error) {
r, err := maxminddb.Open(path)
if err != nil {
return nil, err
}
if err := probeGeoIPReader(r); err != nil {
_ = r.Close()
return nil, err
}
return r, nil
}
// probeGeoIPReader is the functional validation: the well-known
// probe address must resolve to a country code, and a v6 probe must
// at least answer without an error. (maxminddb's Verify is NOT used:
// it also audits metadata niceties — the DB-IP/sapics files carry an
// empty description map and fail it despite being fully readable,
// verified against the factory snapshot on 2026-10-24. A candidate
// that opens and answers both probes is a working country DB; a
// truncated or foreign file fails to open or errors a lookup.)
func probeGeoIPReader(r *maxminddb.Reader) error {
var rec countryRecord
if err := r.Lookup(net.ParseIP(geoipCountryProbeIP), &rec); err != nil {
return err
}
if rec.CountryCode == "" {
return fmt.Errorf("probe address %s resolves to no country", geoipCountryProbeIP)
}
return r.Lookup(net.ParseIP(geoipCountryProbeIPv6), &rec)
}
// verifyGeoIPFile validates a fresh download in place before it may
// replace the live database.
func verifyGeoIPFile(path string) error {
r, err := maxminddb.Open(path)
if err != nil {
return err
}
defer func() { _ = r.Close() }()
return probeGeoIPReader(r)
}
// ---- lookups used by the public views ----
// countryCodeForIP resolves one public IP literal to its ISO country
// code, "" for anything non-public, unknown, or unanswerable. The
// public gate runs first so private/held addresses never reach the
// database (they have no country, exactly as on the hub).
func countryCodeForIP(ipStr string) string {
ipStr = strings.TrimSpace(ipStr)
if !IsPublicIPLiteral(ipStr) {
return ""
}
db := currentGeoIP()
if db == nil || db.reader == nil {
return ""
}
return db.lookup(ipStr)
}
// probeCountryAuto is a node's auto-detected country: the first
// public address in the hub's order — the held interface v4, then
// held v6, then the master-observed exit v4, then exit v6 — resolved
// locally. "" when the node shows no public address or the database
// has no answer for it.
func probeCountryAuto(a *model.AgentNode) string {
for _, s := range []string{a.IPv4, a.IPv6, a.SeenIPv4, a.SeenIPv6} {
if IsPublicIPLiteral(s) {
return countryCodeForIP(s)
}
}
return ""
}
// probeCountry resolves the pair the views publish (hub api.rs): the
// effective country is the operator's pin when one is set, else the
// auto-detected code; country_auto always carries the detected code.
func probeCountry(pin string, a *model.AgentNode) (country, auto string) {
auto = probeCountryAuto(a)
if pin != "" {
return pin, auto
}
return auto, auto
}
// ---- weekly auto-update ----
type geoipFetchResult int
const (
geoipFetchUnchanged geoipFetchResult = iota
geoipFetchUpdated
)
// downloadGeoIPCountry performs one conditional download of url into
// destPath. The candidate lands in a temp file in the destination
// directory, is fully validated there, and only then renames over
// destPath (atomic on the same filesystem); any failure leaves
// destPath and the live database untouched.
func downloadGeoIPCountry(client *http.Client, url, destPath, etag string, ifModifiedSince time.Time) (geoipFetchResult, string, error) {
req, err := http.NewRequest(http.MethodGet, url, nil)
if err != nil {
return geoipFetchUnchanged, etag, err
}
if etag != "" {
req.Header.Set("If-None-Match", etag)
}
if !ifModifiedSince.IsZero() {
req.Header.Set("If-Modified-Since", ifModifiedSince.UTC().Format(http.TimeFormat))
}
resp, err := client.Do(req)
if err != nil {
return geoipFetchUnchanged, etag, err
}
defer func() { _ = resp.Body.Close() }()
if resp.StatusCode == http.StatusNotModified {
return geoipFetchUnchanged, etag, nil
}
if resp.StatusCode != http.StatusOK {
return geoipFetchUnchanged, etag, fmt.Errorf("update server answered %s", resp.Status)
}
dir := filepath.Dir(destPath)
tmp, err := os.CreateTemp(dir, ".geoip-country-*.tmp")
if err != nil {
return geoipFetchUnchanged, etag, err
}
tmpName := tmp.Name()
defer func() { _ = os.Remove(tmpName) }()
n, err := io.Copy(tmp, io.LimitReader(resp.Body, geoipCountryMaxBytes))
if err := tmp.Close(); err != nil {
return geoipFetchUnchanged, etag, err
}
if err != nil {
return geoipFetchUnchanged, etag, err
}
if n == 0 {
return geoipFetchUnchanged, etag, fmt.Errorf("update download was empty")
}
if err := verifyGeoIPFile(tmpName); err != nil {
return geoipFetchUnchanged, etag, fmt.Errorf("downloaded database rejected: %w", err)
}
if err := os.Rename(tmpName, destPath); err != nil {
return geoipFetchUnchanged, etag, err
}
return geoipFetchUpdated, resp.Header.Get("ETag"), nil
}
// UpdateGeoIPCountry runs one update pass: a conditional download
// from the configured URL, a validated atomic replace of the local
// copy, and a hot swap of the live reader. No configured URL or any
// failure keeps the current database; a failure logs a single
// warning (the hub discipline: a broken mirror must not spam).
func UpdateGeoIPCountry() {
url := geoipCountryUpdateURL()
if url == "" {
return
}
dest := GeoIPCountryLocalPath()
var lastMod time.Time
if fi, err := os.Stat(dest); err == nil {
lastMod = fi.ModTime()
}
etag, _ := probeSetting(geoipCountryETagSetting)
client := &http.Client{Timeout: 2 * time.Minute}
res, newETag, err := downloadGeoIPCountry(client, url, dest, etag, lastMod)
if err != nil {
logger.Warningf("geoip: country database update from %s failed: %v (keeping the current database)", url, err)
return
}
if res != geoipFetchUpdated {
return
}
if newETag != "" && newETag != etag {
_ = setProbeSetting(geoipCountryETagSetting, newETag)
}
r, err := openGeoIPFile(dest)
if err != nil {
logger.Warningf("geoip: updated country database %s will not reopen: %v", dest, err)
return
}
swapGeoIP(&geoipDB{reader: r, origin: "local", cache: map[string]string{}})
logger.Infof("geoip: country database updated from %s", url)
}
+7 -2
View File
@@ -288,6 +288,11 @@ func buildProbeNodeView(src probeNodeSource, full bool) map[string]any {
a := src.agent a := src.agent
m := src.meta m := src.meta
t := src.traffic t := src.traffic
// The flag pair (hub api.rs): the effective country is the pin
// when the operator set one, else the locally resolved code for
// the node's public address (geoip_country.go, Q7 — no external
// lookup ever leaves the master).
country, countryAuto := probeCountry(m.CountryPin, &a)
var lastSeenAgo any var lastSeenAgo any
if src.lastSeen > 0 { if src.lastSeen > 0 {
@@ -312,7 +317,7 @@ func buildProbeNodeView(src probeNodeSource, full bool) map[string]any {
view := map[string]any{ view := map[string]any{
"id": a.Id, "id": a.Id,
"name": probeDisplayName(src.name, m.PublicRemark), "name": probeDisplayName(src.name, m.PublicRemark),
"country": m.CountryPin, "country": country,
"group": m.GroupName, "group": m.GroupName,
"public_remark": m.PublicRemark, "public_remark": m.PublicRemark,
"sort": m.Sort, "sort": m.Sort,
@@ -383,7 +388,7 @@ func buildProbeNodeView(src probeNodeSource, full bool) map[string]any {
view["ipv6_auto"] = probeAutoAddress(exits, held4, held6, m.IPv4Pin, "", true) view["ipv6_auto"] = probeAutoAddress(exits, held4, held6, m.IPv4Pin, "", true)
view["interval"] = interval view["interval"] = interval
view["country_pin"] = m.CountryPin view["country_pin"] = m.CountryPin
view["country_auto"] = "" view["country_auto"] = countryAuto
view["remark"] = src.remark view["remark"] = src.remark
view["notify"] = m.Notify view["notify"] = m.Notify
return view return view
+16 -3
View File
@@ -320,9 +320,10 @@ const (
cadenceXrayLogPrune = "@every 10m" cadenceXrayLogPrune = "@every 10m"
// cpu.Percent samples over a full minute (blocking), so a finer cadence just // cpu.Percent samples over a full minute (blocking), so a finer cadence just
// stacks overlapping samplers; subscribers rate-limit alerts to 1/min anyway. // stacks overlapping samplers; subscribers rate-limit alerts to 1/min anyway.
cadenceCheckHash = "@every 2m" cadenceCheckHash = "@every 2m"
cadenceCPUAlarm = "@every 1m" cadenceCPUAlarm = "@every 1m"
cadenceMemoryAlarm = "@every 1m" cadenceMemoryAlarm = "@every 1m"
cadenceGeoIPCountry = "@weekly"
) )
// startTask schedules background jobs (Xray checks, traffic jobs, cron // startTask schedules background jobs (Xray checks, traffic jobs, cron
@@ -378,6 +379,18 @@ func (s *Server) startTask(restartXray bool, loc *time.Location) {
job.NewProbeNotifyHousekeepingJob().Run() job.NewProbeNotifyHousekeepingJob().Run()
}() }()
// Node country flags (Q7): refresh the local country database
// weekly. The pass is a conditional download — a no-op while no
// update URL is configured, cheap (304) while the mirror copy is
// current — and the same one-minute startup kick as the probe
// jobs above lets a long-parked install catch up without waiting
// out the week.
_, _ = s.cron.AddJob(cadenceGeoIPCountry, job.NewGeoIPCountryJob())
go func() {
time.Sleep(time.Minute)
job.NewGeoIPCountryJob().Run()
}()
// check client ips from log file every day // check client ips from log file every day
_, _ = s.cron.AddJob("@daily", job.NewClearLogsJob()) _, _ = s.cron.AddJob("@daily", job.NewClearLogsJob())
_, _ = s.cron.AddJob(cadenceXrayLogPrune, job.NewPruneXrayLogsJob()) _, _ = s.cron.AddJob(cadenceXrayLogPrune, job.NewPruneXrayLogsJob())