Add cloudflare
This commit is contained in:
parent
f66c76ec53
commit
f9265d677a
GPG Key ID:
2DCAF25E55735BFE
|
|
@ -0,0 +1,165 @@
|
|||
__doc__ = 'Downloader/reader for Cloudflare IP ranges'
|
||||
|
||||
from functools import lru_cache
|
||||
from gzip import decompress
|
||||
from http.client import HTTPMessage
|
||||
from ipaddress import IPv4Address, IPv4Network, IPv6Address, IPv6Network
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
from urllib.request import Request, urlopen
|
||||
|
||||
from yaml import dump, load
|
||||
|
||||
from ..config import CONFIG
|
||||
from .my_typing import Ip4DictType, Ip6DictType
|
||||
|
||||
try:
|
||||
from yaml import CDumper as Dumper
|
||||
from yaml import CLoader as Loader
|
||||
except ImportError:
|
||||
from yaml import Dumper, Loader
|
||||
|
||||
# It's most probably the useragent, but python urllib gets denied so we
|
||||
# fake headers
|
||||
_FAKED_HEADERS = {
|
||||
'User-Agent': (
|
||||
'Mozilla/5.0 (Windows NT 10.0; rv:91.0) Gecko/20100101 Firefox/91.0'),
|
||||
'Accept': (
|
||||
'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,'
|
||||
'image/webp,*/*;q=0.8'),
|
||||
'Accept-Language': 'en-US,en;q=0.5',
|
||||
'Accept-Encoding': 'gzip, deflate',
|
||||
'DNT': '1',
|
||||
'Connection': 'close',
|
||||
}
|
||||
|
||||
# https://support.google.com/a/answer/10026322?hl=en
|
||||
_URL_1 = 'https://www.cloudflare.com/ips-v4'
|
||||
_URL_2 = 'https://www.cloudflare.com/ips-v6'
|
||||
_DEFAULT_CFG_V4 = """\
|
||||
# # Format is the following iterable list:
|
||||
# - network: 127.0.0.1/32
|
||||
# reason: First reason
|
||||
# - network: 127.0.0.2/32
|
||||
# reason: second reason
|
||||
"""
|
||||
_DEFAULT_CFG_V6 = """\
|
||||
# # Format is the following iterable list:
|
||||
# - network: 2600:1900::/28
|
||||
# reason: First reason
|
||||
# - network: 2600:1901::/28
|
||||
# reason: second reason
|
||||
"""
|
||||
|
||||
_PATH_MY_DATA_V4 = Path(CONFIG['path_datadir'], 'cloudflare-ipv4.yaml')
|
||||
_PATH_MY_DATA_V6 = Path(CONFIG['path_datadir'], 'cloudflare-ipv6.yaml')
|
||||
_PATH4_MTIME = _PATH6_MTIME = 0
|
||||
_IP4_DICT: Ip4DictType = {}
|
||||
_IP6_DICT: Ip6DictType = {}
|
||||
|
||||
if not _PATH_MY_DATA_V4.is_file():
|
||||
_PATH_MY_DATA_V4.write_text(_DEFAULT_CFG_V4)
|
||||
if not _PATH_MY_DATA_V6.is_file():
|
||||
_PATH_MY_DATA_V6.write_text(_DEFAULT_CFG_V6)
|
||||
|
||||
|
||||
@lru_cache(maxsize=50)
|
||||
def _cloudflare_match_v4_inner(address: IPv4Address) -> Optional[str]:
|
||||
'Try to match an IPv4 address, return data if it matches.'
|
||||
for network in _IP4_DICT:
|
||||
if address in network:
|
||||
return f'Cloudflare: {network} {_IP4_DICT[network]}'
|
||||
|
||||
|
||||
def cloudflare_match_v4(address: IPv4Address) -> Optional[str]:
|
||||
'Reload data when necessary and return result.'
|
||||
if _PATH4_MTIME < _PATH_MY_DATA_V4.stat().st_mtime:
|
||||
_cloudflare_load_v4()
|
||||
return _cloudflare_match_v4_inner(address=address)
|
||||
|
||||
|
||||
@lru_cache(maxsize=50)
|
||||
def _cloudflare_match_v6_inner(address: IPv6Address) -> Optional[str]:
|
||||
'Try to match an IPv6 address, return data if it matches.'
|
||||
for network in _IP6_DICT:
|
||||
if address in network:
|
||||
return f'Cloudflare: {network} {_IP6_DICT[network]}'
|
||||
|
||||
|
||||
def cloudflare_match_v6(address: IPv6Address) -> Optional[str]:
|
||||
'Try to match an IPv6 address, return data if it matches.'
|
||||
if _PATH6_MTIME < _PATH_MY_DATA_V6.stat().st_mtime:
|
||||
_cloudflare_load_v6()
|
||||
return _cloudflare_match_v6_inner(address=address)
|
||||
|
||||
|
||||
def _cloudflare_load_v4():
|
||||
'Load IPv4 data.'
|
||||
global _IP4_DICT, _PATH4_MTIME
|
||||
_PATH4_MTIME = _PATH_MY_DATA_V4.stat().st_mtime
|
||||
my_ip4dict: Ip4DictType = {}
|
||||
with _PATH_MY_DATA_V4.open(mode='r') as fd:
|
||||
data = load(stream=fd, Loader=Loader)
|
||||
if type(data) is list:
|
||||
for item in data:
|
||||
my_ip4dict[IPv4Network(address=item['network'])] = item['reason']
|
||||
_IP4_DICT = my_ip4dict
|
||||
_cloudflare_match_v4_inner.cache_clear()
|
||||
|
||||
|
||||
def _cloudflare_load_v6():
|
||||
'Load IPv6 data.'
|
||||
global _IP6_DICT, _PATH6_MTIME
|
||||
_PATH6_MTIME = _PATH_MY_DATA_V6.stat().st_mtime
|
||||
my_ip6dict: Ip6DictType = {}
|
||||
with _PATH_MY_DATA_V6.open(mode='r') as fd:
|
||||
data = load(stream=fd, Loader=Loader)
|
||||
if type(data) is list:
|
||||
for item in data:
|
||||
my_ip6dict[IPv6Network(address=item['network'])] = item['reason']
|
||||
_IP6_DICT = my_ip6dict
|
||||
_cloudflare_match_v6_inner.cache_clear()
|
||||
|
||||
|
||||
def cloudflare_load():
|
||||
'Load data from files'
|
||||
_cloudflare_load_v4()
|
||||
_cloudflare_load_v6()
|
||||
|
||||
|
||||
def _cloudflare_1_dl(ip4_data: list, ip6_data: list):
|
||||
'Download and parse the first file.'
|
||||
req = Request(url=_URL_1, headers=_FAKED_HEADERS, method='GET')
|
||||
with urlopen(url=req) as fd:
|
||||
data: bytes = fd.read()
|
||||
info: HTTPMessage = fd.info()
|
||||
if info.get('content-encoding') == 'gzip':
|
||||
data: bytes = decompress(data=data)
|
||||
for item in data.decode(encoding='utf-8').split('\n'):
|
||||
ip4_data.append(dict(network=item, reason='Cloudflare IPv4'))
|
||||
|
||||
|
||||
def _cloudflare_2_dl(ip4_data: list, ip6_data: list):
|
||||
'Download and parse the second file.'
|
||||
req = Request(url=_URL_2, headers=_FAKED_HEADERS, method='GET')
|
||||
with urlopen(url=req) as fd:
|
||||
data: bytes = fd.read()
|
||||
info: HTTPMessage = fd.info()
|
||||
if info.get('content-encoding') == 'gzip':
|
||||
data: bytes = decompress(data=data)
|
||||
for item in data.decode(encoding='utf-8').split('\n'):
|
||||
if not item:
|
||||
continue
|
||||
ip6_data.append(dict(network=item, reason='Cloudflare IPv6'))
|
||||
|
||||
|
||||
def cloudflare_download():
|
||||
'Download, parse and save files.'
|
||||
ip4_data = list()
|
||||
ip6_data = list()
|
||||
_cloudflare_1_dl(ip4_data=ip4_data, ip6_data=ip6_data)
|
||||
_cloudflare_2_dl(ip4_data=ip4_data, ip6_data=ip6_data)
|
||||
with _PATH_MY_DATA_V4.open(mode='w') as fd:
|
||||
dump(data=ip4_data, stream=fd, Dumper=Dumper)
|
||||
with _PATH_MY_DATA_V6.open(mode='w') as fd:
|
||||
dump(data=ip6_data, stream=fd, Dumper=Dumper)
|
||||
|
|
@ -6,6 +6,9 @@ from .amazon import (
|
|||
amazon_download, amazon_load, amazon_match_v4, amazon_match_v6)
|
||||
from .asn_checker import reload_asns
|
||||
from .azure import azure_download, azure_load, azure_match_v4, azure_match_v6
|
||||
from .cloudflare import (
|
||||
cloudflare_download, cloudflare_load, cloudflare_match_v4,
|
||||
cloudflare_match_v6)
|
||||
from .custom_blacklist import (
|
||||
_custom_blmatch_v4, _custom_blmatch_v6, custom_load_bl)
|
||||
from .custom_whitelist import (
|
||||
|
|
@ -15,9 +18,11 @@ from .google import (
|
|||
from .useragent import useragent_load
|
||||
|
||||
_MATCH_FUNCS_BL_V4 = [
|
||||
_custom_blmatch_v4, amazon_match_v4, google_match_v4, azure_match_v4]
|
||||
_custom_blmatch_v4, amazon_match_v4, google_match_v4, azure_match_v4,
|
||||
cloudflare_match_v4]
|
||||
_MATCH_FUNCS_BL_V6 = [
|
||||
_custom_blmatch_v6, amazon_match_v6, google_match_v6, azure_match_v6]
|
||||
_custom_blmatch_v6, amazon_match_v6, google_match_v6, azure_match_v6,
|
||||
cloudflare_match_v6]
|
||||
_MATCH_FUNCS_WL_V4 = [_custom_wlmatch_v4]
|
||||
_MATCH_FUNCS_WL_V6 = [_custom_wlmatch_v6]
|
||||
|
||||
|
|
@ -35,6 +40,7 @@ def load():
|
|||
azure_load()
|
||||
custom_load_bl()
|
||||
custom_load_wl()
|
||||
cloudflare_load()
|
||||
reload_asns()
|
||||
useragent_load()
|
||||
|
||||
|
|
@ -44,6 +50,7 @@ def redownload_ranges():
|
|||
amazon_download()
|
||||
google_download()
|
||||
azure_download()
|
||||
cloudflare_download()
|
||||
load()
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue