From b3684efaf62037726968e3a33931f8ae18d2e463 Mon Sep 17 00:00:00 2001
From: bunkerity <contact@bunkerity.com>
Date: Tue, 20 Jul 2021 09:58:09 +0200
Subject: [PATCH] jobs - init work on refactoring

---
 jobs/.Job.py.swp  | Bin 0 -> 1024 bytes
 jobs/ExitNodes.py |  11 ++++++++++
 jobs/Job.py       |  54 ++++++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 65 insertions(+)
 create mode 100644 jobs/.Job.py.swp
 create mode 100644 jobs/ExitNodes.py
 create mode 100644 jobs/Job.py

diff --git a/jobs/.Job.py.swp b/jobs/.Job.py.swp
new file mode 100644
index 0000000000000000000000000000000000000000..497c7c30df39c8f247902708730763cc7c409ccf
GIT binary patch
literal 1024
zcmYc?$V<%2S1{HyVn6{4fV|S;)FKo?92^(dVDAwB0A2SWS7#SL9~`O}81(ZCO7zR~
zi?a2TO7pT)i!!THQ*`svGxI9+v+|RQ^}X_w^a?6*Y9Eyt4S~@RpezJJ(X|5rQB)XF

literal 0
HcmV?d00001

diff --git a/jobs/ExitNodes.py b/jobs/ExitNodes.py
new file mode 100644
index 000000000..6e6091fbf
--- /dev/null
+++ b/jobs/ExitNodes.py
@@ -0,0 +1,11 @@
+from Job import Job
+
+class ExitNodes(Job) :
+
+	def __init__(self, redis_host=None) :
+		name = "ExitNodes"
+		urls = ["https://iplists.firehol.org/files/tor_exits.ipset"]
+		filename = "tor-exit-nodes.list"
+		type = "line"
+		regex = r"^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}/?[0-9]*$"
+		super().__init__(name, urls, filename, redis_host=redis_host, type=type, regex=regex)
diff --git a/jobs/Job.py b/jobs/Job.py
new file mode 100644
index 000000000..33b169e32
--- /dev/null
+++ b/jobs/Job.py
@@ -0,0 +1,54 @@
+import abc, requests, redis, os
+
+class Job(abc.ABC) :
+
+	def __init__(self, name, urls, filename, redis_host=None, type="line", regex=r"^.*$") :
+		self.__name = name
+		self.__urls = urls
+		self.__filename = filename
+		self.__redis = None
+		if redis_host != None :
+			self.__redis = redis.Redis(host=redis_host, port=6379, db=0)
+		self.__type = type
+		self.__regex = regex
+
+	def run(self) :
+		if self.__redis == None :
+			if os.path.isfile("/tmp/" + self.__filename) :
+				os.remove("/tmp/" + self.__filename)
+			file = open("/tmp/" + self.__filename, "a")
+
+		elif self.__redis != None :
+			pipe = self.__redis.pipeline()
+
+		count = 0
+		for url in self.__urls :
+			data = self.__download_data(url)
+			for chunk in data :
+				if self.__type == "line" and not re.match(self.__regex, chunk) :
+					continue
+				count += 1
+				if self.__redis == None :
+					if self.__type == "line" :
+						chunk += b"\n"
+					file.write(chunk)
+				else :
+					pipe.set(self.__name + "_" + chunk, "1")
+
+		if self.__redis == None :
+			file.close()
+			if count > 0 :
+				shutil.copyfile("/tmp/" + self.__filename, "/etc/nginx/" + self.__filename)
+			os.remove("/tmp/" + self.__filename)
+
+		elif self.__redis != None and count > 0 :
+			self.__redis.del(self.__redis.keys(self.__name + "_*"))
+			pipe.execute()
+
+	def __download_data(self, url) :
+		r = requests.get(url, stream=True)
+		if not r or r.status_code != 200 :
+			return False
+		if self.__type == "line" :
+			return r.iter_lines()
+		return r.iter_content(chunk_size=8192)