fix: read comma ranges and colon links

CHANGELOG.md

@@ -1,6 +1,21 @@
 <!-- Changed by tibia.sh in 2026. See "About this copy" in README.md. -->
 # Changelog
 
+## Unreleased
+
+- Read loot ranges with commas as thousands separators, like `1,000-2,000`, as 1000 to 2000 instead of 0 to 2. This
+  covers creature loot and the Loot Statistics pages. A Loot Statistics entry like
+  `Gold Coin, times:1,234, amount:1,000-2,000` keeps `1,234` and `1,000-2,000` whole, and an entry whose `times` has no
+  number is skipped instead of stopping the task. Neither side of a range has a sign, as before, and `1,0002-3` still
+  reads as 2 to 3. No current value changes.
+- Read a currency linked with a leading colon, like `[[:Gold Token]]`, as `Gold Token` instead of `:Gold Token`, in
+  `item.value_buy_currency` and `mount.price_currency`. Colon links to files, images and categories are still skipped.
+  No current value changes.
+- Keep the Loot Statistics task running past a bad page. An entry without `times` is skipped, and a page whose `kills`
+  is 0, missing or not a number gets no drop rows, since no chance can be computed. Before, each of these stopped the
+  task for every later page. A page with negative `kills` also gets no drop rows, and an entry with negative `times`
+  is skipped, instead of replacing a drop with a negative chance. No current value changes.
+
 ## 9.0.0+tibiash.6
 
 - Read numbers with commas as thousands separators, like `50,000`, as the whole number instead of the digits before

tests/test_generation.py

@@ -1,3 +1,4 @@
+# Changed by tibia.sh in 2026. See "About this copy" in README.md.
 import datetime
 import sqlite3
 import unittest
@@ -10,7 +11,7 @@
 from tibiawikisql import generation as generation_module
 from tibiawikisql.api import Article, WikiEntry
 from tibiawikisql.generation import WEAPON_PROFICIENCY_NAME_ARTICLE, WEAPON_PROFICIENCY_TABLES_ARTICLE
-from tibiawikisql.schema import ItemProficiencyPerkTable, ItemTable
+from tibiawikisql.schema import CreatureDropTable, CreatureTable, ItemProficiencyPerkTable, ItemTable
 from tibiawikisql.tasks import images as image_tasks
 from tibiawikisql.tasks.item_proficiency_perks import generate_item_proficiency_perks
 from tibiawikisql.tasks.loot_statistics import generate_loot_statistics
@@ -341,6 +342,121 @@ def test_generate_loot_statistics_early_return_without_maps(self):
         wiki_client.get_articles.assert_not_called()
 
 
+class TestGenerateLootStatistics(unittest.TestCase):
+    DRAGON_PAGE = "{{Loot2\n|kills=200\n|Gold Coin, times:100, amount:1-50\n}}"
+    DRAGON_ROW = (11, 20, 50.0, 1, 50)
+    SEEDED_DEMON_ROW = (10, 20, 12.5, 1, 3)
+
+    def setUp(self):
+        self.conn = sqlite3.connect(":memory:")
+        self.conn.executescript(CreatureTable.get_create_table_statement())
+        self.conn.executescript(CreatureDropTable.get_create_table_statement())
+        timestamp = datetime.datetime.fromisoformat("2024-01-01T00:00:00+00:00")
+        CreatureTable.insert(self.conn, article_id=10, title="Demon", timestamp=timestamp)
+        CreatureTable.insert(self.conn, article_id=11, title="Dragon", timestamp=timestamp)
+        self.data_store = {
+            "creatures_map": {"demon": 10, "dragon": 11},
+            "items_map": {"gold coin": 20, "platinum coin": 21},
+        }
+
+    def tearDown(self):
+        self.conn.close()
+
+    def generate(self, *pages: tuple[str, str]) -> list[tuple]:
+        """Run the task over Loot Statistics pages, given in order as (creature, content) pairs."""
+        wiki_client = Mock()
+        wiki_client.get_articles.return_value = [
+            Article(
+                article_id=9999,
+                title=f"Loot Statistics:{creature}",
+                timestamp=datetime.datetime.fromisoformat("2024-01-01T00:00:00+00:00"),
+                content=content,
+            )
+            for creature, content in pages
+        ]
+        generate_loot_statistics(
+            self.conn,
+            self.data_store,
+            wiki_client=wiki_client,
+            progress_bar=generation_module.progress_bar,
+            article_label=generation_module.article_label,
+            timed=generation_module.timed,
+            echo=Mock(),
+        )
+        return self.conn.execute(
+            "SELECT creature_id, item_id, chance, min, max FROM creature_drop ORDER BY creature_id, item_id",
+        ).fetchall()
+
+    def test_thousands_separators(self):
+        rows = self.generate(("Demon", "{{Loot2\n|kills=2,000\n|Gold Coin, times:1,234, amount:1,000-2,000\n}}"))
+        self.assertEqual([(10, 20, 61.7, 1000, 2000)], rows)
+
+    def test_plain_numbers(self):
+        rows = self.generate(("Demon", "{{Loot2\n|kills=4000\n|Gold Coin, times:3016, amount:1-120\n}}"))
+        self.assertEqual([(10, 20, 75.4, 1, 120)], rows)
+
+    def test_skips_times_without_number(self):
+        rows = self.generate(
+            ("Demon", ("{{Loot2\n|kills=1000\n|Gold Coin, times:abc, amount:1-120\n"
+                       "|Platinum Coin, times:500, amount:1-8\n}}")),
+            ("Dragon", self.DRAGON_PAGE),
+        )
+        self.assertEqual([(10, 21, 50.0, 1, 8), self.DRAGON_ROW], rows)
+
+    def test_skips_entry_without_times(self):
+        rows = self.generate(
+            ("Demon", ("{{Loot2\n|kills=1000\n|Gold Coin, amount:1-120\n"
+                       "|Platinum Coin, times:500, amount:1-8\n}}")),
+            ("Dragon", self.DRAGON_PAGE),
+        )
+        self.assertEqual([(10, 21, 50.0, 1, 8), self.DRAGON_ROW], rows)
+
+    def seed_demon_row(self):
+        """Store a drop the Demon's article gave, which a skipped page must leave in place."""
+        self.conn.execute(
+            "INSERT INTO creature_drop(creature_id, item_id, chance, min, max) VALUES(?,?,?,?,?)",
+            self.SEEDED_DEMON_ROW,
+        )
+
+    def test_skips_page_with_zero_kills(self):
+        self.seed_demon_row()
+        rows = self.generate(
+            ("Demon", "{{Loot2\n|kills=0\n|Gold Coin, times:5, amount:1-120\n}}"),
+            ("Dragon", self.DRAGON_PAGE),
+        )
+        self.assertEqual([self.SEEDED_DEMON_ROW, self.DRAGON_ROW], rows)
+
+    def test_skips_page_with_negative_kills(self):
+        self.seed_demon_row()
+        rows = self.generate(
+            ("Demon", "{{Loot2\n|kills=-100\n|Gold Coin, times:5, amount:1-120\n}}"),
+            ("Dragon", self.DRAGON_PAGE),
+        )
+        self.assertEqual([self.SEEDED_DEMON_ROW, self.DRAGON_ROW], rows)
+
+    def test_keeps_zero_times(self):
+        rows = self.generate(
+            ("Demon", "{{Loot2\n|kills=1000\n|Gold Coin, times:0, amount:1-120\n}}"),
+            ("Dragon", self.DRAGON_PAGE),
+        )
+        self.assertEqual([(10, 20, 0.0, 1, 120), self.DRAGON_ROW], rows)
+
+    def test_skips_negative_times(self):
+        rows = self.generate(
+            ("Demon", ("{{Loot2\n|kills=1000\n|Gold Coin, times:-5, amount:1-120\n"
+                       "|Platinum Coin, times:500, amount:1-8\n}}")),
+            ("Dragon", self.DRAGON_PAGE),
+        )
+        self.assertEqual([(10, 21, 50.0, 1, 8), self.DRAGON_ROW], rows)
+
+    def test_skips_page_without_kills(self):
+        rows = self.generate(
+            ("Demon", "{{Loot2\n|Gold Coin, times:5, amount:1-120\n}}"),
+            ("Dragon", self.DRAGON_PAGE),
+        )
+        self.assertEqual([self.DRAGON_ROW], rows)
+
+
 class TestGenerateCommand(unittest.TestCase):
     def setUp(self):
         self.runner = CliRunner()

tests/tests_utils.py

@@ -116,6 +116,20 @@ def test_parse_currency_link_target(self):
         self.assertEqual("Theons", parse_currency("[[theons]]"))
         self.assertEqual("Silver Token", parse_currency("[[Silver_Token|tokens]]"))
 
+    def test_parse_currency_colon_link(self):
+        for value, expected in (
+            ("[[:Gold Token]]", "Gold Token"),
+            ("[[:theons]]", "Theons"),
+            ("[[ : Gold Token ]]s", "Gold Token"),
+            ("[[:]]", None),
+        ):
+            with self.subTest(value=value):
+                self.assertEqual(expected, parse_currency(value))
+
+    def test_parse_currency_skips_colon_file_and_category_links(self):
+        self.assertIsNone(parse_currency("[[:Category:Foo]]"))
+        self.assertEqual("Theons", parse_currency("[[:File:X.gif]] [[Theons]]"))
+
     def test_parse_currency_section_link_display_text(self):
         self.assertEqual("Hunting Task Points", parse_currency("[[Task Board#Hunting Task Points|Hunting Task Points]]"))
         self.assertEqual("Hunting Task Points", parse_currency("[[Task Board#Hunting Task Points]]"))
@@ -182,6 +196,21 @@ def test_parse_min_max(self):
         self.assertEqual(parse_min_max("5-20"), (5, 20))
         self.assertEqual(parse_min_max("50"), (0, 50))
         self.assertEqual((0, 1000), parse_min_max("1,000"))
+        self.assertEqual((0, 40), parse_min_max("0-40"))
+
+    def test_parse_min_max_thousands_separator(self):
+        for value, expected in (("1,000-2,000", (1000, 2000)), ("0-1,500", (0, 1500))):
+            with self.subTest(value=value):
+                self.assertEqual(expected, parse_min_max(value))
+
+    def test_parse_min_max_not_thousands_separator(self):
+        # No valid thousands number sits next to the hyphen.
+        self.assertEqual((2, 3), parse_min_max("1,0002-3"))
+        self.assertEqual((1, 2), parse_min_max("1-2,0003"))
+
+    def test_parse_min_max_unsigned(self):
+        self.assertEqual((5, 20), parse_min_max("-5-20"))
+        self.assertEqual((0, 5), parse_min_max("5--20"))
 
     def test_parse_sounds(self):
         sound_string = "{{Sound List|Sound 1|Sound 2|Sound 3}}"
@@ -199,6 +228,10 @@ def test_parse_loot_statistics(self):
         kills, _ = parse_loot_statistics(content.replace("|kills=36488", "|kills=1,234"))
         self.assertEqual(1234, kills)
 
+        kills, loot_statistics = parse_loot_statistics(content.replace("|kills=36488", ""))
+        self.assertEqual(0, kills)
+        self.assertEqual(34, len(loot_statistics))
+
         kills, loot_statistics = parse_loot_statistics("Something else")
         self.assertEqual(kills, 0)
         self.assertFalse(loot_statistics)

tibiawikisql/tasks/loot_statistics.py

@@ -1,11 +1,12 @@
+# Changed by tibia.sh in 2026. See "About this copy" in README.md.
 """Task for parsing creature loot statistics pages."""
 from __future__ import annotations
 
 from typing import Any, TYPE_CHECKING
 
 from colorama import Fore, Style
 
-from tibiawikisql.utils import parse_loot_statistics, parse_min_max
+from tibiawikisql.utils import parse_integer, parse_loot_statistics, parse_min_max
 
 if TYPE_CHECKING:
     import sqlite3
@@ -43,6 +44,9 @@ def generate_loot_statistics(
                 if creature_id is None:
                     continue
                 kills, loot_stats = parse_loot_statistics(article.content)
+                if kills <= 0:
+                    # Without a positive number of kills no chance can be computed.
+                    continue
                 rows = []
                 for entry in loot_stats:
                     if not entry:
@@ -52,8 +56,11 @@ def generate_loot_statistics(
                     if item_id is None:
                         unknown_items.add(item_name)
                         continue
+                    times = parse_integer(entry.get("times", ""), None)
+                    if times is None or times < 0:
+                        continue
                     amount = entry.get("amount", 1)
-                    percentage = min(int(entry["times"]) / kills * 100, 100)
+                    percentage = min(times / kills * 100, 100)
                     minimum, maximum = parse_min_max(amount)
                     rows.append((creature_id, item_id, percentage, minimum, maximum))
                     cursor.execute(

tibiawikisql/utils.py

@@ -18,10 +18,16 @@
     from collections.abc import Generator
     from mwparserfromhell.nodes import Template
 
-min_max_pattern = re.compile(r"(\d+)-(\d+)")
+unsigned_number_rule = r"\d{1,3}(?:,\d{3})+(?!\d)|\d+"
+"""The rule for a number without a sign, whose thousands may be grouped with commas, like ``50,000``. Other commas end
+the number."""
+min_max_pattern = re.compile(rf"({unsigned_number_rule})-({unsigned_number_rule})")
+"""A range of two numbers without a sign, separated by a hyphen, like ``0-40`` or ``1,000-2,000``."""
 int_pattern = re.compile(r"[+-]?\d+")
-number_pattern = re.compile(r"[+-]?\d{1,3}(?:,\d{3})+(?!\d)|[+-]?\d+")
+number_pattern = re.compile(rf"[+-]?(?:{unsigned_number_rule})")
 """A number whose thousands may be grouped with commas, like ``50,000``. Other commas end the number."""
+loot_field_separator_pattern = re.compile(r",(?!\d{3}(?!\d))")
+"""A comma between the fields of a loot statistics entry. A comma followed by exactly three digits groups thousands."""
 float_pattern = re.compile(r"[+-]?(\d*[.])?\d+")
 escaping_tags = frozenset(("nowiki", "pre"))
 inert_comment_start = "<!\u200b--"
@@ -375,8 +381,9 @@ def parse_currency(value: str) -> str | None:
     2. Links to files, images and categories are removed.
     3. A value with a remaining link gives the first link's target, the currency's page, so ``[[Silver Token]]s``
        gives ``Silver Token``. When the target is a section of a page, the link's text is used instead, so
-       ``[[Task Board#Hunting Task Points]]`` gives ``Hunting Task Points``. A link that names nothing, like ``[[]]``,
-       gives ``None``.
+       ``[[Task Board#Hunting Task Points]]`` gives ``Hunting Task Points``. A leading colon, which links a page
+       without categorising or embedding it, is removed with the whitespace around it, so ``[[:Gold Token]]`` gives
+       ``Gold Token``. A link that names nothing, like ``[[]]`` or ``[[:]]``, gives ``None``.
     4. Otherwise the text is read without HTML tags and comments. ``gp`` gives :data:`GOLD_COIN`, and ``?`` or no text
        gives ``None``. Any other text is the currency. The contents of ``<nowiki>`` are text, so
        ``<nowiki>{{Foo}}</nowiki>`` gives ``{{Foo}}``.
@@ -400,7 +407,7 @@ def parse_currency(value: str) -> str | None:
         if section:
             text = strip_code(link.text) if link.text is not None else ""
             return text or section.strip() or None
-        name = page.replace("_", " ").strip()
+        name = page.replace("_", " ").strip().removeprefix(":").strip()
         return name[:1].upper() + name[1:] or None
     text = code.strip_code().strip()
     if text == "gp":
@@ -431,13 +438,13 @@ def parse_loot_statistics(value: str) -> tuple[int, list[Any]]:
         value: A string containing a creature's loot statistics.
 
     Returns:
-        A tuple containing the total kills and a list of entries.
+        A tuple containing the total kills and a list of entries. The kills are 0 when missing or not a number.
 
     """
     template = find_template(value, "Loot2", partial=True)
     if not template:
         return 0, []
-    kills = parse_integer(strip_code(template.get("kills", 0)))
+    kills = parse_integer(strip_code(template.get("kills", "")))
     entries = [_parse_loot_entry(param.value.strip_code()) for param in template.params if not param.showkey]
     return kills, entries
 
@@ -525,14 +532,17 @@ def _get_template_param(template: Template, param_name: str) -> str | None:
 def _parse_loot_entry(entry: str) -> dict[str, str]:
     """Parse a single parameter of the loot statistics template.
 
+    Fields are separated by commas. A comma followed by exactly three digits groups thousands, so in
+    ``Gold Coin, times:1,234, amount:1,000-2,000`` the times are ``1,234`` and the amount is ``1,000-2,000``.
+
     Args:
         entry: A single item entry.
 
     Returns:
         A dictionary containing the drop data: item name, times dropped, amount dropped, etcetera.
 
     """
-    arguments = entry.split(",")
+    arguments = loot_field_separator_pattern.split(entry)
     entry = {"amount": "1"}
     for arg in arguments:
         subarg = arg.split(":")
@@ -551,18 +561,20 @@ def _parse_loot_entry(entry: str) -> dict[str, str]:
 def parse_min_max(value: str) -> tuple[int, int]:
     """Parse the minimum and maximum amounts of a loot drop.
 
-    They consist of two numbers separated by a hyphen, e.g. ``0-40``
+    They consist of two numbers separated by a hyphen, e.g. ``0-40``. Thousands may be grouped with commas, so
+    ``1,000-2,000`` gives 1000 and 2000. Neither number has a sign, so ``-5-20`` gives 5 and 20.
 
     Args:
         value: A string containing minimum and maximum values.
 
     Returns:
-        The minimum and maximum amounts.
+        The minimum and maximum amounts. Without a range, the minimum is 0 and the maximum is the first number, or 1 if
+        there is none.
 
     """
     match = min_max_pattern.search(value)
     if match:
-        return int(match.group(1)), int(match.group(2))
+        return int(match.group(1).replace(",", "")), int(match.group(2).replace(",", ""))
     return 0, parse_integer(value, 1)