From f3ceeb73a958e774b1e2fa55d2607cdd3eb419ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophile=20Diot?= Date: Fri, 30 Jun 2023 15:39:04 -0400 Subject: [PATCH] Squashed 'src/deps/src/luajit-geoip/' content from commit fde33e045 git-subtree-dir: src/deps/src/luajit-geoip git-subtree-split: fde33e045083522d73665a6894d78dbf995b9e12 --- Makefile | 20 +++ README.md | 190 ++++++++++++++++++++++++ dist.ini | 9 ++ geoip-dev-1.rockspec | 25 ++++ geoip.lua | 1 + geoip/init.lua | 172 ++++++++++++++++++++++ geoip/init.moon | 115 +++++++++++++++ geoip/mmdb.lua | 341 +++++++++++++++++++++++++++++++++++++++++++ geoip/mmdb.moon | 337 ++++++++++++++++++++++++++++++++++++++++++ geoip/version.lua | 1 + geoip/version.moon | 1 + spec/geoip_spec.moon | 22 +++ spec/mmdb_spec.moon | 210 ++++++++++++++++++++++++++ 13 files changed, 1444 insertions(+) create mode 100644 Makefile create mode 100644 README.md create mode 100644 dist.ini create mode 100644 geoip-dev-1.rockspec create mode 100644 geoip.lua create mode 100644 geoip/init.lua create mode 100644 geoip/init.moon create mode 100644 geoip/mmdb.lua create mode 100644 geoip/mmdb.moon create mode 100644 geoip/version.lua create mode 100644 geoip/version.moon create mode 100644 spec/geoip_spec.moon create mode 100644 spec/mmdb_spec.moon diff --git a/Makefile b/Makefile new file mode 100644 index 000000000..4cd52fc18 --- /dev/null +++ b/Makefile @@ -0,0 +1,20 @@ + +.PHONY: test local build valgrind + +test: + busted + +local: build + luarocks make --lua-version=5.1 --local geoip-dev-1.rockspec + +build: + moonc geoip + +valgrind_geoip: + valgrind --leak-check=yes --trace-children=yes busted spec/geoip_spec.moon + +valgrind_mmdb: + valgrind --leak-check=yes --trace-children=yes busted spec/mmdb_spec.moon + +lint:: + git ls-files | grep '\.moon$$' | xargs -n 100 moonc -l diff --git a/README.md b/README.md new file mode 100644 index 000000000..073ecfca5 --- /dev/null +++ b/README.md @@ -0,0 +1,190 @@ + +# LuaJIT bindings to MaxMind's GeoIP and GeoIP2 (libmaxminddb) libraries + +* https://github.com/maxmind/libmaxminddb +* https://github.com/maxmind/geoip-api-c — legacy library + +In order to use this library you'll need LuaJIT, the GeoIP library you're +trying to use, and the databases files for the appropriate library. You should +be able to find these in your package manager. + +**I recommend using libmaxminddb**, as the legacy GeoIP databases are no +longer updated. + +## Install + +```bash +luarocks install --server=http://luarocks.org/manifests/leafo geoip +``` + +# Reference + +## libmaxminddb + +The module is named `geoip.mmdb` + +```lua +local geoip = require "geoip.mmdb" +``` + +This module works great in OpenResty. You'll want to keep references to loaded +GeoIP DB objects at the module level, in order avoid reloading the DB on every +request. + +
+ +See OpenResty example + +Create a new module for your GeoIP databases: + +**`geoip_helper.lua`** + +```lua +local geoip = require "geoip.mmdb" + +return { + country_db = assert(geoip.load_database("/var/lib/GeoIP/GeoLite2-Country.mmdb")), + -- load more databases if necessary: + -- asnum_db = ... + -- etc. +} +``` + +**OpenResty request handler:** + +```lua +-- this module will be cached in `package.loaded`, and the databases will only be loaded on first access +local result = require("geoip_helper").country_db.lookup_addr(ngx.var.remote_addr) +if result then + ngx.say("Your country:" .. result.country.iso_code) +end +``` + +> **Note:** If you're using a proxy with x-forwarded-for you'll need to adjust +> how you access the user's IP address + +
+ + +### `db, err = load_database(file_name)` + +Load the database from the file path. Returns `nil` and error message if the +database could not be loaded. + +The location of database files vary depending on the system and type of +database. For this example we'll use the country database located at +`/var/lib/GeoIP/GeoLite2-Country.mmdb`. + + +```lua +local mmdb = assert(geoip.load_database("/var/lib/GeoIP/GeoLite2-Country.mmdb")) +``` + +The database object has the following methods: + + +### `object, err = mmdb:lookup(address)` + +```lua +local result = assert(mmdb:lookup("8.8.8.8")) + +-- print the country code +print(result.country.iso_code) --> US +``` + +Look up an address (as a string), and return all data about it as a Lua table. +Returns `nil` and an error if the address could not be looked up, or there was +no information for that address. + +> Note: You can lookup both ipv4 and ipv6 addresses + +The structure of the output depends on the database used. (It matches the +structure of the out from the `mmdblookup` utility, if you need a quick way to +check) + +### `value, err = mmdb:lookup_value(address, ...)` + +```lua +-- prints the country code +print(assert(mmdb:lookup_value("8.8.8.8", "country", "iso_code"))) --> US +``` + +Looks up a single value for an address using the path specified in the varargs +`...`. Returns `nil` and an error if the address is invalid or a value was not +located at the path. This method avoids scanning the entire object for an +address's entry, so it may be more efficient if a specific value from the +database is needed. + + +## geoip — legacy + +*The databases for this library are no longer updated, I strongly recommend +using the mmdb functionality above* + +The module is named `geoip` + +```lua +local geoip = require "geoip" +``` + +GeoIP has support for many different database types. The available lookup +databases are automatically loaded from the system location. + +Only the country and ASNUM databases are supported. Feel free to create a pull +request with support for more. + +### `res, err = lookup_addr(ip_address)` + +Look up information about an address. Returns an table with properties about +that address extracted from all available databases. + + +```lua +local geoip = require "geoip" +local res = geoip.lookup_addr("8.8.8.8") + +print(res.country_code) +``` + +The structure of the return value looks like this: + +```lua +{ + country_code = "US", + country_name = "United States", + asnum = "AS15169 Google Inc." +} +``` + +### Controlling database caching + +You can control how the databases are loaded by manually instantiating a +`GeoIP` object and calling the `load_databases` method directly. `lookup_addr` +will automatically load databases only if they haven't been loaded yet. + +```lua +local geoip = require("geoip") + +local gi = geoip.GeoIP() +gi:load_databases("memory") + +local res = gi:lookup_addr("8.8.8.8") +``` + +> By default the STANDARD mode is used, which reads from disk for each lookup + + +# Version history + + +* **2.1** *(Aug 28, 2020)* — Fix bug with parsing booleans from mmdb ([#3](https://github.com/leafo/luajit-geoip/pull/3)) michaeljmartin +* **2.0** *(Apr 6, 2020)* — Support for mmdb (libmaxminddb), fix memory leak in geoip +* **1.0** *(Apr 4, 2018)* — Initial release, support for geoip + +# Contact + +License: MIT, Copyright 2020 +Author: Leaf Corcoran (leafo) ([@moonscript](http://twitter.com/moonscript)) +Email: leafot@gmail.com +Homepage: + diff --git a/dist.ini b/dist.ini new file mode 100644 index 000000000..61caba469 --- /dev/null +++ b/dist.ini @@ -0,0 +1,9 @@ +name=geoip +abstract=LuaJIT bindings to MaxMind GeoIP +author=Leaf Corcoran (leafo) +is_original=yes +license=mit +lib_dir=. +doc_dir=. +repo_link=https://github.com/leafo/luajit-geoip +main_module=geoip/init.lua \ No newline at end of file diff --git a/geoip-dev-1.rockspec b/geoip-dev-1.rockspec new file mode 100644 index 000000000..973e71f2a --- /dev/null +++ b/geoip-dev-1.rockspec @@ -0,0 +1,25 @@ +package = "geoip" +version = "dev-1" + +source = { + url = "git://github.com/leafo/luajit-geoip.git", +} + +description = { + summary = "LuaJIT bindings to MaxMind GeoIP library", + license = "MIT", + maintainer = "Leaf Corcoran ", +} + +dependencies = { + "lua == 5.1", +} + +build = { + type = "builtin", + modules = { + ["geoip"] = "geoip/init.lua", + ["geoip.mmdb"] = "geoip/mmdb.lua", + ["geoip.version"] = "geoip/version.lua", + } +} diff --git a/geoip.lua b/geoip.lua new file mode 100644 index 000000000..8255887fb --- /dev/null +++ b/geoip.lua @@ -0,0 +1 @@ +return require "geoip.init" diff --git a/geoip/init.lua b/geoip/init.lua new file mode 100644 index 000000000..371100ce6 --- /dev/null +++ b/geoip/init.lua @@ -0,0 +1,172 @@ +local ffi = require("ffi") +local bit = require("bit") +ffi.cdef([[ typedef struct GeoIP {} GeoIP; + + typedef enum { + GEOIP_STANDARD = 0, + GEOIP_MEMORY_CACHE = 1, + GEOIP_CHECK_CACHE = 2, + GEOIP_INDEX_CACHE = 4, + GEOIP_MMAP_CACHE = 8, + GEOIP_SILENCE = 16, + } GeoIPOptions; + + typedef enum { + GEOIP_COUNTRY_EDITION = 1, + GEOIP_CITY_EDITION_REV1 = 2, + GEOIP_ASNUM_EDITION = 9, + } GeoIPDBTypes; + + typedef enum { + GEOIP_CHARSET_ISO_8859_1 = 0, + GEOIP_CHARSET_UTF8 = 1 + } GeoIPCharset; + + int GeoIP_db_avail(int type); + GeoIP * GeoIP_open_type(int type, int flags); + + void GeoIP_delete(GeoIP * gi); + char *GeoIP_database_info(GeoIP * gi); + + int GeoIP_charset(GeoIP * gi); + int GeoIP_set_charset(GeoIP * gi, int charset); + + unsigned long _GeoIP_lookupaddress(const char *host); + + char *GeoIP_name_by_addr(GeoIP * gi, const char *addr); + int GeoIP_id_by_addr(GeoIP * gi, const char *addr); + + unsigned GeoIP_num_countries(void); + const char * GeoIP_code_by_id(int id); + const char * GeoIP_country_name_by_id(GeoIP * gi, int id); +]]) +local lib = ffi.load("GeoIP") +local DATABASE_TYPES = { + lib.GEOIP_COUNTRY_EDITION, + lib.GEOIP_ASNUM_EDITION +} +local CACHE_TYPES = { + standard = lib.GEOIP_STANDARD, + memory = lib.GEOIP_MEMORY_CACHE, + check = lib.GEOIP_CHECK_CACHE, + index = lib.GEOIP_INDEX_CACHE +} +local GeoIP +do + local _class_0 + local _base_0 = { + load_databases = function(self, mode) + if mode == nil then + mode = lib.GEOIP_STANDARD + end + mode = CACHE_TYPES[mode] or mode + if self.databases then + return + end + do + local _accum_0 = { } + local _len_0 = 1 + for _index_0 = 1, #DATABASE_TYPES do + local _continue_0 = false + repeat + local i = DATABASE_TYPES[_index_0] + if not (1 == lib.GeoIP_db_avail(i)) then + _continue_0 = true + break + end + local gi = lib.GeoIP_open_type(i, bit.bor(mode, lib.GEOIP_SILENCE)) + if gi == nil then + _continue_0 = true + break + end + ffi.gc(gi, (assert(lib.GeoIP_delete, "missing destructor"))) + lib.GeoIP_set_charset(gi, lib.GEOIP_CHARSET_UTF8) + local _value_0 = { + type = i, + gi = gi + } + _accum_0[_len_0] = _value_0 + _len_0 = _len_0 + 1 + _continue_0 = true + until true + if not _continue_0 then + break + end + end + self.databases = _accum_0 + end + return true + end, + country_by_id = function(self, gi, id) + if id < 0 or id >= lib.GeoIP_num_countries() then + return + end + local code = lib.GeoIP_code_by_id(id) + local country = lib.GeoIP_country_name_by_id(gi, id) + code = code ~= nil and ffi.string(code) or nil + country = country ~= nil and ffi.string(country) or nil + if code == "--" then + code = nil + end + return code, country + end, + lookup_addr = function(self, ip) + self:load_databases() + local out = { } + local _list_0 = self.databases + for _index_0 = 1, #_list_0 do + local _continue_0 = false + repeat + local _des_0 = _list_0[_index_0] + local type, gi + type, gi = _des_0.type, _des_0.gi + local _exp_0 = type + if lib.GEOIP_COUNTRY_EDITION == _exp_0 then + local cid = lib.GeoIP_id_by_addr(gi, ip) + out.country_code, out.country_name = self:country_by_id(gi, cid) + elseif lib.GEOIP_ASNUM_EDITION == _exp_0 then + local asnum = lib.GeoIP_name_by_addr(gi, ip) + if asnum == nil then + _continue_0 = true + break + end + out.asnum = ffi.string(asnum) + end + _continue_0 = true + until true + if not _continue_0 then + break + end + end + if next(out) then + return out + end + end + } + _base_0.__index = _base_0 + _class_0 = setmetatable({ + __init = function(self) end, + __base = _base_0, + __name = "GeoIP" + }, { + __index = _base_0, + __call = function(cls, ...) + local _self_0 = setmetatable({}, _base_0) + cls.__init(_self_0, ...) + return _self_0 + end + }) + _base_0.__class = _class_0 + GeoIP = _class_0 +end +return { + GeoIP = GeoIP, + lookup_addr = (function() + local _base_0 = GeoIP() + local _fn_0 = _base_0.lookup_addr + return function(...) + return _fn_0(_base_0, ...) + end + end)(), + VERSION = require("geoip.version") +} diff --git a/geoip/init.moon b/geoip/init.moon new file mode 100644 index 000000000..e2a4a4950 --- /dev/null +++ b/geoip/init.moon @@ -0,0 +1,115 @@ +ffi = require "ffi" +bit = require "bit" + +ffi.cdef [[ + typedef struct GeoIP {} GeoIP; + + typedef enum { + GEOIP_STANDARD = 0, + GEOIP_MEMORY_CACHE = 1, + GEOIP_CHECK_CACHE = 2, + GEOIP_INDEX_CACHE = 4, + GEOIP_MMAP_CACHE = 8, + GEOIP_SILENCE = 16, + } GeoIPOptions; + + typedef enum { + GEOIP_COUNTRY_EDITION = 1, + GEOIP_CITY_EDITION_REV1 = 2, + GEOIP_ASNUM_EDITION = 9, + } GeoIPDBTypes; + + typedef enum { + GEOIP_CHARSET_ISO_8859_1 = 0, + GEOIP_CHARSET_UTF8 = 1 + } GeoIPCharset; + + int GeoIP_db_avail(int type); + GeoIP * GeoIP_open_type(int type, int flags); + + void GeoIP_delete(GeoIP * gi); + char *GeoIP_database_info(GeoIP * gi); + + int GeoIP_charset(GeoIP * gi); + int GeoIP_set_charset(GeoIP * gi, int charset); + + unsigned long _GeoIP_lookupaddress(const char *host); + + char *GeoIP_name_by_addr(GeoIP * gi, const char *addr); + int GeoIP_id_by_addr(GeoIP * gi, const char *addr); + + unsigned GeoIP_num_countries(void); + const char * GeoIP_code_by_id(int id); + const char * GeoIP_country_name_by_id(GeoIP * gi, int id); +]] + +lib = ffi.load "GeoIP" + +DATABASE_TYPES = { + lib.GEOIP_COUNTRY_EDITION + lib.GEOIP_ASNUM_EDITION +} + +CACHE_TYPES = { + standard: lib.GEOIP_STANDARD + memory: lib.GEOIP_MEMORY_CACHE + check: lib.GEOIP_CHECK_CACHE + index: lib.GEOIP_INDEX_CACHE +} + +class GeoIP + new: => + + load_databases: (mode=lib.GEOIP_STANDARD) => + mode = CACHE_TYPES[mode] or mode + return if @databases + @databases = for i in *DATABASE_TYPES + continue unless 1 == lib.GeoIP_db_avail(i) + + gi = lib.GeoIP_open_type i, bit.bor mode, lib.GEOIP_SILENCE + continue if gi == nil + ffi.gc gi, (assert lib.GeoIP_delete, "missing destructor") + lib.GeoIP_set_charset gi, lib.GEOIP_CHARSET_UTF8 + + { + type: i + :gi + } + + true + + country_by_id: (gi, id) => + if id < 0 or id >= lib.GeoIP_num_countries! + return + + code = lib.GeoIP_code_by_id id + country = lib.GeoIP_country_name_by_id gi, id + + code = code != nil and ffi.string(code) or nil + country = country != nil and ffi.string(country) or nil + code = nil if code == "--" + + code, country + + lookup_addr: (ip) => + @load_databases! + + out = {} + for {:type, :gi} in *@databases + switch type + when lib.GEOIP_COUNTRY_EDITION + cid = lib.GeoIP_id_by_addr gi, ip + out.country_code, out.country_name = @country_by_id gi, cid + when lib.GEOIP_ASNUM_EDITION + asnum = lib.GeoIP_name_by_addr gi, ip + continue if asnum == nil + out.asnum = ffi.string asnum + + out if next out + +{ + :GeoIP + lookup_addr: GeoIP!\lookup_addr + VERSION: require "geoip.version" +} + diff --git a/geoip/mmdb.lua b/geoip/mmdb.lua new file mode 100644 index 000000000..9050f0bd7 --- /dev/null +++ b/geoip/mmdb.lua @@ -0,0 +1,341 @@ +local ffi = require("ffi") +local bit = require("bit") +local MMDB_MODE_MMAP = 1 +local MMDB_MODE_MASK = 7 +local MMDB_SUCCESS = 0 +local MMDB_FILE_OPEN_ERROR = 1 +local MMDB_CORRUPT_SEARCH_TREE_ERROR = 2 +local MMDB_INVALID_METADATA_ERROR = 3 +local MMDB_IO_ERROR = 4 +local MMDB_OUT_OF_MEMORY_ERROR = 5 +local MMDB_UNKNOWN_DATABASE_FORMAT_ERROR = 6 +local MMDB_INVALID_DATA_ERROR = 7 +local MMDB_INVALID_LOOKUP_PATH_ERROR = 8 +local MMDB_LOOKUP_PATH_DOES_NOT_MATCH_DATA_ERROR = 9 +local MMDB_INVALID_NODE_NUMBER_ERROR = 10 +local MMDB_IPV6_LOOKUP_IN_IPV4_DATABASE_ERROR = 11 +local DATA_TYPES = { + MMDB_DATA_TYPE_EXTENDED = 0, + MMDB_DATA_TYPE_POINTER = 1, + MMDB_DATA_TYPE_UTF8_STRING = 2, + MMDB_DATA_TYPE_DOUBLE = 3, + MMDB_DATA_TYPE_BYTES = 4, + MMDB_DATA_TYPE_UINT16 = 5, + MMDB_DATA_TYPE_UINT32 = 6, + MMDB_DATA_TYPE_MAP = 7, + MMDB_DATA_TYPE_INT32 = 8, + MMDB_DATA_TYPE_UINT64 = 9, + MMDB_DATA_TYPE_UINT128 = 10, + MMDB_DATA_TYPE_ARRAY = 11, + MMDB_DATA_TYPE_CONTAINER = 12, + MMDB_DATA_TYPE_END_MARKER = 13, + MMDB_DATA_TYPE_BOOLEAN = 14, + MMDB_DATA_TYPE_FLOAT = 15 +} +local _list_0 +do + local _accum_0 = { } + local _len_0 = 1 + for k in pairs(DATA_TYPES) do + _accum_0[_len_0] = k + _len_0 = _len_0 + 1 + end + _list_0 = _accum_0 +end +for _index_0 = 1, #_list_0 do + local key = _list_0[_index_0] + DATA_TYPES[DATA_TYPES[key]] = key +end +ffi.cdef([[ const char *gai_strerror(int ecode); + + typedef unsigned int mmdb_uint128_t __attribute__ ((__mode__(TI))); + + typedef struct MMDB_entry_s { + const struct MMDB_s *mmdb; + uint32_t offset; + } MMDB_entry_s; + + typedef struct MMDB_lookup_result_s { + bool found_entry; + MMDB_entry_s entry; + uint16_t netmask; + } MMDB_lookup_result_s; + + + typedef struct MMDB_entry_data_s { + bool has_data; + union { + uint32_t pointer; + const char *utf8_string; + double double_value; + const uint8_t *bytes; + uint16_t uint16; + uint32_t uint32; + int32_t int32; + uint64_t uint64; + mmdb_uint128_t uint128; + bool boolean; + float float_value; + }; + /* This is a 0 if a given entry cannot be found. This can only happen + * when a call to MMDB_(v)get_value() asks for hash keys or array + * indices that don't exist. */ + uint32_t offset; + /* This is the next entry in the data section, but it's really only + * relevant for entries that part of a larger map or array + * struct. There's no good reason for an end user to look at this + * directly. */ + uint32_t offset_to_next; + /* This is only valid for strings, utf8_strings or binary data */ + uint32_t data_size; + /* This is an MMDB_DATA_TYPE_* constant */ + uint32_t type; + } MMDB_entry_data_s; + + typedef struct MMDB_entry_data_list_s { + MMDB_entry_data_s entry_data; + struct MMDB_entry_data_list_s *next; + void *pool; + } MMDB_entry_data_list_s; + + typedef struct MMDB_description_s { + const char *language; + const char *description; + } MMDB_description_s; + + typedef struct MMDB_metadata_s { + uint32_t node_count; + uint16_t record_size; + uint16_t ip_version; + const char *database_type; + struct { + size_t count; + const char **names; + } languages; + uint16_t binary_format_major_version; + uint16_t binary_format_minor_version; + uint64_t build_epoch; + struct { + size_t count; + MMDB_description_s **descriptions; + } description; + /* See above warning before adding fields */ + } MMDB_metadata_s; + + typedef struct MMDB_ipv4_start_node_s { + uint16_t netmask; + uint32_t node_value; + /* See above warning before adding fields */ + } MMDB_ipv4_start_node_s; + + typedef struct MMDB_s { + uint32_t flags; + const char *filename; + ssize_t file_size; + const uint8_t *file_content; + const uint8_t *data_section; + uint32_t data_section_size; + const uint8_t *metadata_section; + uint32_t metadata_section_size; + uint16_t full_record_byte_size; + uint16_t depth; + MMDB_ipv4_start_node_s ipv4_start_node; + MMDB_metadata_s metadata; + /* See above warning before adding fields */ + } MMDB_s; + + extern int MMDB_open(const char *const filename, uint32_t flags, + MMDB_s *const mmdb); + + extern void MMDB_close(MMDB_s *const mmdb); + + extern MMDB_lookup_result_s MMDB_lookup_string(const MMDB_s *const mmdb, + const char *const ipstr, + int *const gai_error, + int *const mmdb_error); + + extern const char *MMDB_strerror(int error_code); + + extern int MMDB_get_entry_data_list( + MMDB_entry_s *start, MMDB_entry_data_list_s **const entry_data_list); + + extern void MMDB_free_entry_data_list( + MMDB_entry_data_list_s *const entry_data_list); + + extern int MMDB_get_value(MMDB_entry_s *const start, + MMDB_entry_data_s *const entry_data, + ...); +]]) +local lib = ffi.load("libmaxminddb") +local consume_map, consume_array +local consume_value +consume_value = function(current) + if current == nil then + return nil, "expected value but go nothing" + end + local entry_data = current.entry_data + local _exp_0 = entry_data.type + if DATA_TYPES.MMDB_DATA_TYPE_MAP == _exp_0 then + return assert(consume_map(current)) + elseif DATA_TYPES.MMDB_DATA_TYPE_ARRAY == _exp_0 then + return assert(consume_array(current)) + elseif DATA_TYPES.MMDB_DATA_TYPE_UTF8_STRING == _exp_0 then + local value = ffi.string(entry_data.utf8_string, entry_data.data_size) + return value, current.next + elseif DATA_TYPES.MMDB_DATA_TYPE_UINT32 == _exp_0 then + local value = entry_data.uint32 + return value, current.next + elseif DATA_TYPES.MMDB_DATA_TYPE_UINT16 == _exp_0 then + local value = entry_data.uint16 + return value, current.next + elseif DATA_TYPES.MMDB_DATA_TYPE_INT32 == _exp_0 then + local value = entry_data.int32 + return value, current.next + elseif DATA_TYPES.MMDB_DATA_TYPE_UINT64 == _exp_0 then + local value = entry_data.uint64 + return value, current.next + elseif DATA_TYPES.MMDB_DATA_TYPE_DOUBLE == _exp_0 then + local value = entry_data.double_value + return value, current.next + elseif DATA_TYPES.MMDB_DATA_TYPE_BOOLEAN == _exp_0 then + assert(entry_data.boolean ~= nil) + local value = entry_data.boolean + return value, current.next + else + error("unknown type: " .. tostring(DATA_TYPES[entry_data.type])) + return nil, current.next + end +end +consume_map = function(current) + local out = { } + local map = current.entry_data + local tuple_count = map.data_size + current = current.next + while tuple_count > 0 do + local key + key, current = assert(consume_value(current)) + local value + value, current = consume_value(current) + out[key] = value + tuple_count = tuple_count - 1 + end + return out, current +end +consume_array = function(current) + local out = { } + local array = current.entry_data + local length = array.data_size + current = current.next + while length > 0 do + local value + value, current = assert(consume_value(current)) + table.insert(out, value) + length = length - 1 + end + return out, current +end +local Mmdb +do + local _class_0 + local _base_0 = { + load = function(self) + self.mmdb = ffi.new("MMDB_s") + local res = lib.MMDB_open(self.file_path, 0, self.mmdb) + if not (res == MMDB_SUCCESS) then + return nil, "failed to load db: " .. tostring(self.file_path) + end + ffi.gc(self.mmdb, (assert(lib.MMDB_close, "missing destructor"))) + return true + end, + _lookup_string = function(self, ip) + assert(self.mmdb, "mmdb database is not loaded") + local gai_error = ffi.new("int[1]") + local mmdb_error = ffi.new("int[1]") + local res = lib.MMDB_lookup_string(self.mmdb, ip, gai_error, mmdb_error) + if not (gai_error[0] == MMDB_SUCCESS) then + return nil, "gai error: " .. tostring(ffi.string(lib.gai_strerror(gai_error[0]))) + end + if not (mmdb_error[0] == MMDB_SUCCESS) then + return nil, "mmdb error: " .. tostring(ffi.string(lib.MMDB_strerror(mmdb_error[0]))) + end + if not (res.found_entry) then + return nil, "failed to find entry" + end + return res + end, + lookup_value = function(self, ip, ...) + assert((...), "missing path") + local path = { + ... + } + table.insert(path, 0) + local res, err = self:_lookup_string(ip) + if not (res) then + return nil, err + end + local entry_data = ffi.new("MMDB_entry_data_s") + local status = lib.MMDB_get_value(res.entry, entry_data, unpack(path)) + if MMDB_SUCCESS ~= status then + return nil, "failed to find field by path" + end + if entry_data.has_data then + local _exp_0 = entry_data.type + if DATA_TYPES.MMDB_DATA_TYPE_MAP == _exp_0 or DATA_TYPES.MMDB_DATA_TYPE_ARRAY == _exp_0 then + return nil, "path holds object, not value" + end + local value = assert(consume_value({ + entry_data = entry_data + })) + return value + else + return nil, "entry has no data" + end + end, + lookup = function(self, ip) + local res, err = self:_lookup_string(ip) + if not (res) then + return nil, err + end + local entry_data_list = ffi.new("MMDB_entry_data_list_s*[1]") + local status = lib.MMDB_get_entry_data_list(res.entry, entry_data_list) + if not (status == MMDB_SUCCESS) then + return nil, "failed to load data: " .. tostring(ffi.string(lib.MMDB_strerror(status))) + end + ffi.gc(entry_data_list[0], (assert(lib.MMDB_free_entry_data_list, "missing destructor"))) + local current = entry_data_list[0] + local value = assert(consume_value(current)) + return value + end + } + _base_0.__index = _base_0 + _class_0 = setmetatable({ + __init = function(self, file_path, opts) + self.file_path, self.opts = file_path, opts + end, + __base = _base_0, + __name = "Mmdb" + }, { + __index = _base_0, + __call = function(cls, ...) + local _self_0 = setmetatable({}, _base_0) + cls.__init(_self_0, ...) + return _self_0 + end + }) + _base_0.__class = _class_0 + Mmdb = _class_0 +end +local load_database +load_database = function(filename) + local mmdb = Mmdb(filename) + local success, err = mmdb:load() + if not (success) then + return nil, err + end + return mmdb +end +return { + Mmdb = Mmdb, + load_database = load_database, + VERSION = require("geoip.version") +} diff --git a/geoip/mmdb.moon b/geoip/mmdb.moon new file mode 100644 index 000000000..91ac10006 --- /dev/null +++ b/geoip/mmdb.moon @@ -0,0 +1,337 @@ +ffi = require "ffi" +bit = require "bit" + +-- extracted from /usr/include/maxminddb.h + +-- flags for open +MMDB_MODE_MMAP = 1 +MMDB_MODE_MASK = 7 + +-- error codes +MMDB_SUCCESS = 0 +MMDB_FILE_OPEN_ERROR = 1 +MMDB_CORRUPT_SEARCH_TREE_ERROR = 2 +MMDB_INVALID_METADATA_ERROR = 3 +MMDB_IO_ERROR = 4 +MMDB_OUT_OF_MEMORY_ERROR = 5 +MMDB_UNKNOWN_DATABASE_FORMAT_ERROR = 6 +MMDB_INVALID_DATA_ERROR = 7 +MMDB_INVALID_LOOKUP_PATH_ERROR = 8 +MMDB_LOOKUP_PATH_DOES_NOT_MATCH_DATA_ERROR = 9 +MMDB_INVALID_NODE_NUMBER_ERROR = 10 +MMDB_IPV6_LOOKUP_IN_IPV4_DATABASE_ERROR = 11 + +-- data types +DATA_TYPES = { + MMDB_DATA_TYPE_EXTENDED: 0 + MMDB_DATA_TYPE_POINTER: 1 + MMDB_DATA_TYPE_UTF8_STRING: 2 + MMDB_DATA_TYPE_DOUBLE: 3 + MMDB_DATA_TYPE_BYTES: 4 + MMDB_DATA_TYPE_UINT16: 5 + MMDB_DATA_TYPE_UINT32: 6 + MMDB_DATA_TYPE_MAP: 7 + MMDB_DATA_TYPE_INT32: 8 + MMDB_DATA_TYPE_UINT64: 9 + MMDB_DATA_TYPE_UINT128: 10 + MMDB_DATA_TYPE_ARRAY: 11 + MMDB_DATA_TYPE_CONTAINER: 12 + MMDB_DATA_TYPE_END_MARKER: 13 + MMDB_DATA_TYPE_BOOLEAN: 14 + MMDB_DATA_TYPE_FLOAT: 15 +} + +for key in *[k for k in pairs DATA_TYPES] + DATA_TYPES[DATA_TYPES[key]] = key + +ffi.cdef [[ + const char *gai_strerror(int ecode); + + typedef unsigned int mmdb_uint128_t __attribute__ ((__mode__(TI))); + + typedef struct MMDB_entry_s { + const struct MMDB_s *mmdb; + uint32_t offset; + } MMDB_entry_s; + + typedef struct MMDB_lookup_result_s { + bool found_entry; + MMDB_entry_s entry; + uint16_t netmask; + } MMDB_lookup_result_s; + + + typedef struct MMDB_entry_data_s { + bool has_data; + union { + uint32_t pointer; + const char *utf8_string; + double double_value; + const uint8_t *bytes; + uint16_t uint16; + uint32_t uint32; + int32_t int32; + uint64_t uint64; + mmdb_uint128_t uint128; + bool boolean; + float float_value; + }; + /* This is a 0 if a given entry cannot be found. This can only happen + * when a call to MMDB_(v)get_value() asks for hash keys or array + * indices that don't exist. */ + uint32_t offset; + /* This is the next entry in the data section, but it's really only + * relevant for entries that part of a larger map or array + * struct. There's no good reason for an end user to look at this + * directly. */ + uint32_t offset_to_next; + /* This is only valid for strings, utf8_strings or binary data */ + uint32_t data_size; + /* This is an MMDB_DATA_TYPE_* constant */ + uint32_t type; + } MMDB_entry_data_s; + + typedef struct MMDB_entry_data_list_s { + MMDB_entry_data_s entry_data; + struct MMDB_entry_data_list_s *next; + void *pool; + } MMDB_entry_data_list_s; + + typedef struct MMDB_description_s { + const char *language; + const char *description; + } MMDB_description_s; + + typedef struct MMDB_metadata_s { + uint32_t node_count; + uint16_t record_size; + uint16_t ip_version; + const char *database_type; + struct { + size_t count; + const char **names; + } languages; + uint16_t binary_format_major_version; + uint16_t binary_format_minor_version; + uint64_t build_epoch; + struct { + size_t count; + MMDB_description_s **descriptions; + } description; + /* See above warning before adding fields */ + } MMDB_metadata_s; + + typedef struct MMDB_ipv4_start_node_s { + uint16_t netmask; + uint32_t node_value; + /* See above warning before adding fields */ + } MMDB_ipv4_start_node_s; + + typedef struct MMDB_s { + uint32_t flags; + const char *filename; + ssize_t file_size; + const uint8_t *file_content; + const uint8_t *data_section; + uint32_t data_section_size; + const uint8_t *metadata_section; + uint32_t metadata_section_size; + uint16_t full_record_byte_size; + uint16_t depth; + MMDB_ipv4_start_node_s ipv4_start_node; + MMDB_metadata_s metadata; + /* See above warning before adding fields */ + } MMDB_s; + + extern int MMDB_open(const char *const filename, uint32_t flags, + MMDB_s *const mmdb); + + extern void MMDB_close(MMDB_s *const mmdb); + + extern MMDB_lookup_result_s MMDB_lookup_string(const MMDB_s *const mmdb, + const char *const ipstr, + int *const gai_error, + int *const mmdb_error); + + extern const char *MMDB_strerror(int error_code); + + extern int MMDB_get_entry_data_list( + MMDB_entry_s *start, MMDB_entry_data_list_s **const entry_data_list); + + extern void MMDB_free_entry_data_list( + MMDB_entry_data_list_s *const entry_data_list); + + extern int MMDB_get_value(MMDB_entry_s *const start, + MMDB_entry_data_s *const entry_data, + ...); +]] + +lib = ffi.load "libmaxminddb" + +local consume_map, consume_array + +consume_value = (current) -> + if current == nil + return nil, "expected value but go nothing" + + entry_data = current.entry_data + + switch entry_data.type + when DATA_TYPES.MMDB_DATA_TYPE_MAP + assert consume_map current + when DATA_TYPES.MMDB_DATA_TYPE_ARRAY + assert consume_array current + when DATA_TYPES.MMDB_DATA_TYPE_UTF8_STRING + value = ffi.string entry_data.utf8_string, entry_data.data_size + value, current.next + when DATA_TYPES.MMDB_DATA_TYPE_UINT32 + value = entry_data.uint32 + value, current.next + when DATA_TYPES.MMDB_DATA_TYPE_UINT16 + value = entry_data.uint16 + value, current.next + when DATA_TYPES.MMDB_DATA_TYPE_INT32 + value = entry_data.int32 + value, current.next + when DATA_TYPES.MMDB_DATA_TYPE_UINT64 + value = entry_data.uint64 + value, current.next + when DATA_TYPES.MMDB_DATA_TYPE_DOUBLE + value = entry_data.double_value + value, current.next + when DATA_TYPES.MMDB_DATA_TYPE_BOOLEAN + assert entry_data.boolean ~= nil + value = entry_data.boolean + value, current.next + else + error "unknown type: #{DATA_TYPES[entry_data.type]}" + nil, current.next + +consume_map = (current) -> + out = {} + + map = current.entry_data + tuple_count = map.data_size + + -- move to first value + current = current.next + + while tuple_count > 0 + key, current = assert consume_value current + value, current = consume_value current + out[key] = value + tuple_count -= 1 + + out, current + +consume_array = (current) -> + out = {} + + array = current.entry_data + length = array.data_size + + -- move to first value + current = current.next + + while length > 0 + value, current = assert consume_value current + table.insert out, value + length -= 1 + + out, current + +class Mmdb + new: (@file_path, @opts) => + + load: => + @mmdb = ffi.new "MMDB_s" + res = lib.MMDB_open @file_path, 0, @mmdb + + unless res == MMDB_SUCCESS + return nil, "failed to load db: #{@file_path}" + + ffi.gc @mmdb, (assert lib.MMDB_close, "missing destructor") + true + + _lookup_string: (ip) => + assert @mmdb, "mmdb database is not loaded" + + gai_error = ffi.new "int[1]" + mmdb_error = ffi.new "int[1]" + + res = lib.MMDB_lookup_string @mmdb, ip, gai_error, mmdb_error + + unless gai_error[0] == MMDB_SUCCESS + return nil, "gai error: #{ffi.string lib.gai_strerror gai_error[0]}" + + unless mmdb_error[0] == MMDB_SUCCESS + return nil, "mmdb error: #{ffi.string lib.MMDB_strerror mmdb_error[0]}" + + unless res.found_entry + return nil, "failed to find entry" + + res + + lookup_value: (ip, ...) => + assert (...), "missing path" + path = {...} + table.insert path, 0 + + res, err = @_lookup_string ip + unless res + return nil, err + + entry_data = ffi.new "MMDB_entry_data_s" + + status = lib.MMDB_get_value res.entry, entry_data, unpack path + + if MMDB_SUCCESS != status + return nil, "failed to find field by path" + + if entry_data.has_data + -- the node we get don't have the data so we have to bail if path leads + -- to a map or array + switch entry_data.type + when DATA_TYPES.MMDB_DATA_TYPE_MAP, DATA_TYPES.MMDB_DATA_TYPE_ARRAY + return nil, "path holds object, not value" + + value = assert consume_value { + :entry_data + } + + value + else + nil, "entry has no data" + + lookup: (ip) => + res, err = @_lookup_string ip + + unless res + return nil, err + + entry_data_list = ffi.new "MMDB_entry_data_list_s*[1]" + + status = lib.MMDB_get_entry_data_list res.entry, entry_data_list + + unless status == MMDB_SUCCESS + return nil, "failed to load data: #{ffi.string lib.MMDB_strerror status}" + + ffi.gc entry_data_list[0], (assert lib.MMDB_free_entry_data_list, "missing destructor") + + current = entry_data_list[0] + value = assert consume_value current + + value + +load_database = (filename) -> + mmdb = Mmdb filename + success, err = mmdb\load! + unless success + return nil, err + mmdb + +{ + :Mmdb + :load_database + VERSION: require "geoip.version" +} diff --git a/geoip/version.lua b/geoip/version.lua new file mode 100644 index 000000000..999a17b32 --- /dev/null +++ b/geoip/version.lua @@ -0,0 +1 @@ +return "2.1.0" diff --git a/geoip/version.moon b/geoip/version.moon new file mode 100644 index 000000000..8320adbf0 --- /dev/null +++ b/geoip/version.moon @@ -0,0 +1 @@ +"2.1.0" diff --git a/spec/geoip_spec.moon b/spec/geoip_spec.moon new file mode 100644 index 000000000..a32099772 --- /dev/null +++ b/spec/geoip_spec.moon @@ -0,0 +1,22 @@ + +import lookup_addr from require "geoip" + +describe "geoip", -> + it "looks up address", -> + assert.same { + asnum: "AS15169 GOOGLE" + country_code: "US" + country_name: "United States" + }, lookup_addr "8.8.8.8" + + it "looks up bad address", -> + assert.same nil, (lookup_addr "helloo.world") + + + it "manually instantiates database with memory lookup", -> + import GeoIP from require "geoip" + geoip = GeoIP! + geoip\load_databases "memory" + assert.truthy lookup_addr "8.8.8.8" + + diff --git a/spec/mmdb_spec.moon b/spec/mmdb_spec.moon new file mode 100644 index 000000000..3970ea622 --- /dev/null +++ b/spec/mmdb_spec.moon @@ -0,0 +1,210 @@ + +country_db = "/var/lib/GeoIP/GeoLite2-Country.mmdb" +city_db = "/var/lib/GeoIP/GeoLite2-City.mmdb" +asnum_db = "/var/lib/GeoIP/GeoLite2-ASN.mmdb" + +mmdb = require "geoip.mmdb" + +describe "mmdb", -> + it "handles invalid database path", -> + assert.same {nil, "failed to load db: hello.world.db"}, { + mmdb.load_database "hello.world.db" + } + + it "handles invalid database file", -> + assert.same {nil, "failed to load db: README.md"}, { + mmdb.load_database "README.md" + } + + describe "asnum_db", -> + local db + before_each -> + db = assert mmdb.load_database asnum_db + + it "looks up address", -> + out = assert db\lookup "1.1.1.1" + assert.same { + autonomous_system_organization: "CLOUDFLARENET" + autonomous_system_number: 13335 + }, out + + it "looks up localhost", -> + assert.same {nil, "failed to find entry"}, {db\lookup "127.0.0.1"} + + it "looks up invalid address", -> + assert.same { + nil, "gai error: Name or service not known" + }, {db\lookup "efjlewfk"} + + it "looks up ipv6", -> + assert.same { + autonomous_system_number: 15169 + autonomous_system_organization: "GOOGLE" + }, db\lookup "2001:4860:4860::8888" + + describe "country_db", -> + local db + before_each -> + db = assert mmdb.load_database country_db + + it "looks up address", -> + out = assert db\lookup "8.8.8.8" + assert.same { + continent: { + code: 'NA' + geoname_id: 6255149 + names: { + "de": 'Nordamerika' + "en": 'North America' + "es": 'Norteamérica' + "fr": 'Amérique du Nord' + "ja": '北アメリカ' + "pt-BR": 'América do Norte' + "ru": 'Северная Америка' + "zh-CN": '北美洲' + } + } + country: { + geoname_id: 6252001 + iso_code: 'US' + names: { + "de": 'USA' + "en": 'United States' + "es": 'Estados Unidos' + "fr": 'États-Unis' + "ja": 'アメリカ合衆国' + "pt-BR": 'Estados Unidos' + "ru": 'США' + "zh-CN": '美国' + } + } + registered_country: { + geoname_id: 6252001 + iso_code: 'US' + names: { + "de": 'USA' + "en": 'United States' + "es": 'Estados Unidos' + "fr": 'États-Unis' + "ja": 'アメリカ合衆国' + "pt-BR": 'Estados Unidos' + "ru": 'США' + "zh-CN": '美国' + } + } + }, out + + it "looks up EU address 212.237.134.97", -> + out = assert db\lookup "212.237.134.97" + assert.same { + continent: { + code: 'EU' + geoname_id: 6255148 + names: { + "de": 'Europa' + "en": 'Europe' + "es": 'Europa' + "fr": 'Europe' + "ja": 'ヨーロッパ' + "pt-BR": 'Europa' + "ru": 'Европа' + "zh-CN": '欧洲' + } + } + country: { + geoname_id: 2623032 + is_in_european_union: true + iso_code: 'DK' + names: { + "de": 'Dänemark' + "en": 'Denmark' + "es": 'Dinamarca' + "fr": 'Danemark' + "ja": 'デンマーク王国' + "pt-BR": 'Dinamarca' + "ru": 'Дания' + "zh-CN": '丹麦' + } + } + registered_country: { + geoname_id: 2623032 + is_in_european_union: true + iso_code: 'DK' + names: { + "de": 'Dänemark' + "en": 'Denmark' + "es": 'Dinamarca' + "fr": 'Danemark' + "ja": 'デンマーク王国' + "pt-BR": 'Dinamarca' + "ru": 'Дания' + "zh-CN": '丹麦' + } + } + }, out + + describe "lookup_value", -> + it "looks up string value", -> + res = assert db\lookup_value "8.8.8.8", "country", "iso_code" + assert.same "US", res + + it "looks up number value", -> + res = assert db\lookup_value "8.8.8.8", "continent", "geoname_id" + assert.same 6255149, res + + it "handles looking up invalid path", -> + assert.same {nil, "failed to find field by path"}, { + db\lookup_value "8.8.8.8", "continent", "fart" + } + + it "handles missing path", -> + assert.has_error( + -> db\lookup_value "8.8.8.8" + "missing path" + ) + + it "handles invalid root", -> + assert.same {nil, "failed to find field by path"}, { + db\lookup_value "8.8.8.8", "butt" + } + + it "returning object field", -> + assert.same {nil, "path holds object, not value"}, { + db\lookup_value "8.8.8.8", "continent" + } + + describe "city_db", -> + local db + before_each -> + db = assert mmdb.load_database city_db + + it "looks up address", -> + out = assert db\lookup "1.1.1.1" + assert.same { + accuracy_radius: 1000 + longitude: 143.2104 + latitude: -33.494 + time_zone: "Australia/Sydney" + }, out.location + + it "looks up address with subdivisions (an array)", -> + out = assert db\lookup "173.255.250.29" + assert.same { + { + names: { + "en": "California" + "zh-CN": "加利福尼亚州" + "fr": "Californie" + "ru": "Калифорния" + "es": "California" + "pt-BR": "Califórnia" + "de": "Kalifornien" + "ja": "カリフォルニア州" + } + iso_code: "CA" + geoname_id: 5332921 + } + }, out.subdivisions + + assert.same "94536", out.postal.code +