dynamic tracker lists: Gtk support + refactoring

We now avoid duplicate trackers, when using multiple lists.

The new Gtk text entry widget - one that supports multiple lines - has a
different API, because of course it does...

Author: stefantalpalaru

defaulttrackers/core.py

@@ -86,28 +86,36 @@ def update_trackerlist_from_url(self):
             now = datetime.datetime.utcnow()
             last_update = datetime.datetime.utcfromtimestamp(self.config["last_dynamic_trackers_update"])
             if now - last_update > datetime.timedelta(days=self.config["dynamic_trackers_update_interval"]):
-                combined_trackers = []
+                new_trackers = []
+                seen_trackers = set()
+                headers = {
+                    'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:62.0) Gecko/20100101 Firefox/62.0',
+                    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+                    'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.3',
+                    'Accept-Encoding': 'none',
+                    'Accept-Language': 'en-US,en;q=0.8',
+                }
                 try:
                     for url_line in self.config["dynamic_trackerlist_url"].splitlines():
                         url_line = url_line.strip()
                         if not url_line:
                             continue
-                        headers = {
-                            'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:62.0) Gecko/20100101 Firefox/62.0',
-                            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
-                            'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.3',
-                            'Accept-Encoding': 'none',
-                            'Accept-Language': 'en-US,en;q=0.8',
-                        }
                         req = urllib.request.Request(url_line, headers=headers)
                         try:
                             page = urllib.request.urlopen(req, context=ssl._create_unverified_context()).read()
                         except:
+                            # maybe an older Python version without a "context" argument
                             page = urllib.request.urlopen(req).read()
-                        found = [decode_bytes(u) for u in re.findall(rb'\w+://[\w\-.:/]+', page) if is_url(decode_bytes(u))]
-                        combined_trackers.extend(found)
-                    if combined_trackers:
-                        self.config["trackers"] = [{"url": t} for t in combined_trackers]
+                        # we need to avoid duplicates here
+                        for new_tracker in [decode_bytes(u) for u in re.findall(rb'\w+://[\w\-.:/]+', page) if is_url(decode_bytes(u))]:
+                            if new_tracker not in seen_trackers:
+                                seen_trackers.add(new_tracker)
+                                new_trackers.append(new_tracker)
+
+                    if new_trackers:
+                        # replace all existing trackers
+                        self.config["trackers"] = [{"url": t} for t in new_trackers]
+
                     self.config["last_dynamic_trackers_update"] = time.mktime(now.timetuple())
                 except:
                     traceback.print_exc()

defaulttrackers/data/config.glade

@@ -40,14 +40,9 @@
                           </packing>
                         </child>
                         <child>
-                          <widget class="GtkEntry" id="tracker_list_url">
+                          <widget class="GtkTextView" id="tracker_list_url">
                             <property name="visible">True</property>
                             <property name="can_focus">True</property>
-                            <property name="invisible_char">●</property>
-                            <property name="primary_icon_activatable">False</property>
-                            <property name="secondary_icon_activatable">False</property>
-                            <property name="primary_icon_sensitive">True</property>
-                            <property name="secondary_icon_sensitive">True</property>
                           </widget>
                           <packing>
                             <property name="expand">True</property>

defaulttrackers/data/config.ui

@@ -40,14 +40,9 @@
                           </packing>
                         </child>
                         <child>
-                          <object class="GtkEntry" id="tracker_list_url">
+                          <object class="GtkTextView" id="tracker_list_url">
                             <property name="visible">True</property>
                             <property name="can_focus">True</property>
-                            <property name="invisible_char">●</property>
-                            <property name="primary_icon_activatable">False</property>
-                            <property name="secondary_icon_activatable">False</property>
-                            <property name="primary_icon_sensitive">True</property>
-                            <property name="secondary_icon_sensitive">True</property>
                           </object>
                           <packing>
                             <property name="expand">True</property>

defaulttrackers/gtk3ui.py

@@ -181,9 +181,12 @@ def on_apply_prefs(self):
             update_interval = int(self.builder.get_object("tracker_list_update_interval").get_text() or 1)
         except:
             update_interval = 1
+        tracker_list_buffer = self.builder.get_object("tracker_list_url").get_buffer()
+        start_iter, end_iter = tracker_list_buffer.get_bounds()
+        tracker_list_text = tracker_list_buffer.get_text(start_iter, end_iter, False)
         self.config.update({
             "trackers": [{"url": str(row[0])} for row in self.store],
-            "dynamic_trackerlist_url": self.builder.get_object("tracker_list_url").get_text(),
+            "dynamic_trackerlist_url": tracker_list_text,
             "dynamic_trackers_update_interval": update_interval,
         })
         client.defaulttrackers.set_config(self.config)
@@ -196,7 +199,7 @@ def cb_get_config(self, config):
         self.config = config
         # dynamic tracker list
         if config["dynamic_trackerlist_url"]:
-            self.builder.get_object("tracker_list_url").set_text(config["dynamic_trackerlist_url"])
+            self.builder.get_object("tracker_list_url").get_buffer().set_text(config["dynamic_trackerlist_url"])
         self.builder.get_object("tracker_list_update_interval").set_text(str(config["dynamic_trackers_update_interval"]))
         # trackers
         self.trackers = list(config["trackers"])

defaulttrackers/gtkui.py

@@ -178,9 +178,12 @@ def on_apply_prefs(self):
             update_interval = int(self.glade.get_widget("tracker_list_update_interval").get_text() or 1)
         except:
             update_interval = 1
+        tracker_list_buffer = self.glade.get_widget("tracker_list_url").get_buffer()
+        start_iter, end_iter = tracker_list_buffer.get_bounds()
+        tracker_list_text = tracker_list_buffer.get_text(start_iter, end_iter, False)
         self.config.update({
             "trackers": [{"url": str(row[0])} for row in self.store],
-            "dynamic_trackerlist_url": self.glade.get_widget("tracker_list_url").get_text(),
+            "dynamic_trackerlist_url": tracker_list_text,
             "dynamic_trackers_update_interval": update_interval,
         })
         client.defaulttrackers.set_config(self.config)
@@ -193,7 +196,7 @@ def cb_get_config(self, config):
         self.config = config
         # dynamic tracker list
         if config["dynamic_trackerlist_url"]:
-            self.glade.get_widget("tracker_list_url").set_text(config["dynamic_trackerlist_url"])
+            self.glade.get_widget("tracker_list_url").get_buffer().set_text(config["dynamic_trackerlist_url"])
         self.glade.get_widget("tracker_list_update_interval").set_text(str(config["dynamic_trackers_update_interval"]))
         # trackers
         self.trackers = list(config["trackers"])