powerdns: update GeoIP backend configuration

With the recent powerdns upgrade, the GeoIP backend uses a new config
format, see zones-file.yml.jinja2.

zz.countries.nexedi.dk.rbldnsd (which maps IPs to countries) is replaced
by GeoLite2-Country.mmdb(1) (which maps IPs to countries & continents).

Note: zz.countries.nexedi.dk.rbldnsd is not deleted (yet?); it is parsed
to keep information about China and its IP ranges grouped by ISP.

cdn.conf.in (which maps RR(s) to countries) is dropped; it is replaced
by an equivalent mapping done inside zones-file.yml.jinja2.

Also, EDNS Client Subnet extension is enabled.

---
(1): Added in previous commit.

software/powerdns/buildout.hash.cfg

@@ -18,11 +18,11 @@ md5sum = 863fdbee998fd4f69fe359667768e33c
 
 [template-powerdns]
 filename = instance-powerdns.cfg
-md5sum = 708e52e8161d64e6d826c630cfa70d5c
+md5sum = 2adb91323d60fc350f52910a3257d4a7
 
 [template-pdns-configuration]
 _update_hash_filename_ = template/pdns.conf.jinja2
-md5sum = 8f3e1bd21480f5e9d8cefa33d4527516
+md5sum = e45d72de87b4adb89c195ba463be4077
 
 [template-dns-replicate]
 _update_hash_filename_ = instance-powerdns-replicate.cfg.jinja2
@@ -32,6 +32,6 @@ md5sum = 670bac1a8463e08814db82a81ddae1bf
 _update_hash_filename_ = template/zz.countries.nexedi.dk.rbldnsd
 md5sum = c4dc8c141d81b92d92cdb82ca67a13ee
 
-[template-cdn-conf]
-_update_hash_filename_ = template/cdn.conf.in
-md5sum = 29c29f93b3b0bd2f71f86f7b337e4543
+[template-zones-file]
+_update_hash_filename_ = template/zones-file.yml.jinja2
+md5sum = c820a4f53c3e7706f51a5e0be3a8cf74

software/powerdns/instance-powerdns.cfg

 {% if slap_software_type in software_type -%}
-{% set part_list =  [] %}
 
 # Create all needed directories
 [directory]
@@ -63,14 +62,12 @@ socket-directory = $${pdns-directory:socket}
 webserver-port = 8088
 
 [geo]
-ip-map-zonefile = ${iso-list:target}
-geo-maps = $${pdns-directory:geo-maps}
+zones-file = $${zones-file-template:rendered}
 database = ${geolite2-country:location}/GeoLite2-Country.mmdb
 
 [pdns-directory]
 recipe = slapos.cookbook:mkdirectory
 configuration = $${directory:etc}/pdns
-geo-maps = $${:configuration}/geo-maps
 socket = $${directory:run}/pdns-socket
 
 [pdns-configuration-template]
@@ -81,6 +78,39 @@ extra-context =
   section pdns pdns
   section geo geo
 
+[asia]
+japan = jp
+hong-kong = hk
+china-telecom = cn-t
+china-unicom = cn-u
+china-mobile = cn-m
+west-asia = ae af am az bh cc cy ge il iq ir jo kg kw kz lb om pk qa ru sa sy tj tm tr uz ye
+east-asia = bn bt cx id in io kh kp kr la lk mm mn mo mv my np ph sg th to tw vn
+
+[china]
+recipe = slapos.recipe.build
+iso-list = ${iso-list:target}
+init =
+  import re
+  ip_split = []
+  match = re.compile(r"(.*) :.*:(cn-\w)\n").fullmatch
+  with open(options["iso-list"]) as f:
+    for line in f:
+      m = match(line)
+      if m is None:
+        continue
+      ip_split.append(m.groups())
+  options["ip-split"] = ip_split
+
+[zones-file-template]
+< = jinja2-template-base
+template = ${template-zones-file:target}
+extensions = jinja2.ext.do
+rendered = $${pdns-directory:configuration}/zones-file.yml
+extra-context =
+  section asia asia
+  key china china:ip-split
+
 # Executables
 [pdns-server]
 recipe = slapos.cookbook:wrapper
@@ -118,29 +148,6 @@ monitor-url = $${monitor-publish-parameters:monitor-url}
 monitor-user = $${monitor-publish-parameters:monitor-user}
 monitor-password = $${monitor-publish-parameters:monitor-password}
 
-#####################
-# Power DNS Slave configuration
-#
-{% set slave_instance_list = json_module.loads(slapparameter_dict.get('extra_slave_instance_list', '[]')) %}
-
-# Iter through slave list to prepare configuration
-{% for slave in slave_instance_list %}
-{%   if 'record' in slave and 'origin' in slave and 'default' in slave %}
-{%     set slave_reference = slave.get('slave_reference') %}
-{%     set slave_section_name = 'map-configuration-%s' % slave_reference  %}
-{%     do part_list.append(slave_section_name) %}
-[{{ slave_section_name }}]
-< = jinja2-template-base
-template = ${template-cdn-conf:target}
-rendered = $${geo:geo-maps}/{{ slave_reference }}
-configuration = {{ json_module.dumps(slave) }}
-extra-context =
-  key json_cdn :configuration
-{%   endif %}
-{% endfor %}
-
-####################
-
 [buildout]
 parts =
   pdns-configuration-template
@@ -150,9 +157,6 @@ parts =
   pdns-promise-listen-port
   monitor-base
   publish-connection-informations
-{% for part in part_list %}
-{{ '  %s' % part }}
-{% endfor %}
 
 extends = ${monitor-template:output}
 

software/powerdns/software.cfg

@@ -45,7 +45,7 @@ recipe = slapos.recipe.build:download
 url = ${:_profile_base_location_}/${:_update_hash_filename_}
 mode = 0644
 
-[template-cdn-conf]
+[template-zones-file]
 recipe = slapos.recipe.build:download
 url = ${:_profile_base_location_}/${:_update_hash_filename_}
 mode = 0644

software/powerdns/template/pdns.conf.jinja2

@@ -31,7 +31,8 @@ cache-ttl=0
 # things are working. :)
 log-dns-details=yes
 log-dns-queries=yes
-log-failed-updates=yes
+# https://github.com/PowerDNS/pdns/commit/df9d980
+# log-failed-updates=yes
 loglevel=4
 
 # This disables wildcards which is more efficient.  geobackend doesn't use
@@ -40,54 +41,11 @@ loglevel=4
 # wildcards=no
 
 # The geobackend
-launch=geo
+launch=geoip
 
-# The zone that your geo-balanced RR is inside of.  The whole zone has to be
-# delegated to the PowerDNS backend, so you will generally want to make up some
-# subzone of your main zone.  We chose "geo.blitzed.org".
-#
-geo-zone={{ slapparameter_dict.get('zone', 'example.com') }}
-
-# The only parts of the SOA for "geo.blitzed.org" that apply here are the
-# master server name and the contact address.
-geo-soa-values={{ slapparameter_dict.get('soa', 'ns0.example.com,admin@example.com') }}
-
-# List of NS records of the PowerDNS servers that are authoritative for your
-# GLB zone.
-geo-ns-records={{ slapparameter_dict.get('ns-record', 'ns0.example.com,ns1.example.com') }}
-
-# The TTL of the CNAME records that geobackend will return.  Since the same
-# resolver will always get the same CNAME (apart from if the director-map
-# changes) it is safe to return a reasonable TTL, so if you leave this
-# commented then a sane default will be chosen.
-#geo-ttl=3600
-
-# The TTL of the NS records that will be returned.  Leave this commented if you
-# don't understand.
-#geo-ns-ttl=86400
-
-# This is the real guts of the data that drives this backend.  This is a DNS
-# zone file for RBLDNSD, a nameserver specialised for running large DNS zones
-# typical of DNSBLs and such.  We choose it for our data because it is easier
-# to parse than the BIND-format one.
-#
-# Anyway, it comes from http://countries.nerd.dk/more.html - there are details
-# there for how to rsync your own copy.  You'll want to do that regularly,
-# every couple of days maybe.  We believe the nerd.dk guys take the netblock
-# info from Regional Internet Registries (RIRs) like RIPE, ARIN, APNIC.  From
-# that they build a big zonefile of IP/prefixlen -> ISO-country-code mappings.
-geo-ip-map-zonefile={{  geo.get('ip-map-zonefile') }}
-
-# And finally this last directive tells the geobackend where to find the map
-# files that say a) which RR to answer for, and b) what actual resource record
-# to return for each ISO country code.  The setting here is a comma-separated
-# list of paths, each of which may either be a single map file or a directory
-# that will contain map files.  If you are only ever going to serve one RR then
-# a single file is probably better, but if you're going to serve many then a
-# directory would probably be better.  The rest of this documentation will
-# assume you chose a directory.
-geo-maps={{  geo.get('geo-maps') }}
+edns-subnet-processing=yes
 
 geoip-database-files={{ geo['database'] }}
+geoip-zones-file={{ geo['zones-file'] }}
 
 # -------------------------------------------------------------------------
\ No newline at end of file

software/powerdns/template/zones-file.yml.jinja2

+# See https://doc.powerdns.com/authoritative/backends/geoip.html
+
+{%- set slave_instance_list = json_module.loads(slapparameter_dict.get('extra_slave_instance_list', '[]')) %}
+{%- set zone = slapparameter_dict.get('zone', 'example.com') %}
+
+{%- macro disambiguate_domain_name(a, b) %}
+{#- See http://www.dns-sd.org/trailingdotsindomainnames.html #}
+{%- if a.endswith('.') %}
+{{- a[:-1] }}
+{%- else %}
+{{- a }}.{{ b }}
+{%- endif %}
+{%- endmacro %}
+
+domains:
+- domain: {{ zone }}
+  # TODO: what value for ttl?
+  ttl: 300
+
+  # Note: For each domain, one record of the domain name MUST exist with a soa record.
+  records:
+    {{ zone }}:
+      - soa: {{ slapparameter_dict.get('soa', 'ns0.example.com,admin@example.com').replace(',', ' ') }}
+      {%- for ns in slapparameter_dict.get('ns-record', 'ns0.example.com,ns1.example.com').split(',') %}
+      - ns: {{ ns }}
+      {%- endfor %}
+
+    {#- Split the world the way we prefer. 'GeoLite2-Country.mmdb' divides the
+        world into 7 continents: Oceania, Asia, Europe, South America,
+        North America, Africa and Antarctica. However, we also want a more
+        fine-grained division for Asia, i.e. West/East, China, Hong-Kong and
+        Japan. #}
+    {%- set world_split = {
+          "continent": {
+            "eu": "europe",
+            "af": "africa",
+            "sa": "south-america",
+            "na": "north-america",
+            "oc": "oceania",
+          },
+          "country": {}} %}
+
+    {%- for region, codes in asia.items() %}
+    {%-   for country_code in codes.split() %}
+    {%-     do world_split["country"].__setitem__(country_code, region) %}
+    {%-   endfor %}
+    {%- endfor %}
+
+    {%- for slave in slave_instance_list %}
+
+    {#-   Set the RR to use for each region, as described in
+          'slave-instance-powerdns-input-schema.json' #}
+    {%-   set rr_dict = {} %}
+    {%-   for region, default_rr in {"europe": "eu",
+                                     "africa": "af",
+                                     "south-america": "sa",
+                                     "north-america": "na",
+                                     "china-telecom": "cn-t",
+                                     "china-unicom": "cn-u",
+                                     "china-mobile": "cn-m",
+                                     "japan": "jp",
+                                     "hong-kong": "hk",
+                                     "east-asia": "as",
+                                     "west-asia": "eu",
+                                     "oceania": "oc"}.items() %}
+    {%-     do rr_dict.__setitem__(region, slave.get(region, default_rr)) %}
+    {%-   endfor %}
+
+    {#-   'code2region' maps an ISO3166 country/continent code (i.e. the client
+          origin) to a geographical region. The latter is then used to find the
+          right RR (thanks to 'rr_dict') #}
+    {%-   for placeholder, code2region in world_split.items() %}
+    {%-     for code, region in code2region.items() %}
+    {%-       set origin = slave['origin'] %}
+    {{ code }}.{{ placeholder }}.{{ origin }}:
+      - cname: {{ disambiguate_domain_name(rr_dict[region], origin) }}
+    {%-     endfor %}
+    {%-   endfor %}
+    {%- endfor %}
+
+  services:
+    {%- for slave in slave_instance_list %}
+    {%-   set origin = slave['origin'] %}
+    {{ disambiguate_domain_name(slave['record'], zone) }}:
+      {#- Note: Placeholders (i.e. "country." and "continent.") are used to avoid
+          possible name collisions, e.g.:
+          - %cc for American Samoa is 'as'
+          - %cn for Asia is also 'as' #}
+      default: ['%cc.country.{{ origin }}', '%cn.continent.{{ origin }}', '{{ disambiguate_domain_name(slave['default'], origin) }}']
+
+      # Split China's ip addresses according to ISP
+      {%- for ip_range, country_code in china %}
+      {{ ip_range }}: {{ country_code }}.country.{{ origin }}
+      {%- endfor %}
+    {%- endfor %}