From 866120e18e4acb1a4409ad436f8d9db9a4f7f7fd Mon Sep 17 00:00:00 2001 From: eminsk Date: Wed, 30 Sep 2026 15:59:43 -0400 Subject: [PATCH] feat: track price changes (#214) and extract seller name (#333) --- .github/workflows/ci.yml | 35 +++++++++ db_service.py | 30 +++++++- integrations/notifications/base.py | 16 +++- integrations/notifications/vk.py | 15 +++- load_config.py | 5 +- models.py | 26 ++++--- parser/export/excel.py | 6 +- parser_cls.py | 42 +++++++++- tests/test_excel_export.py | 20 +++++ tests/test_price_change.py | 118 +++++++++++++++++++++++++++++ tests/test_seller_info.py | 44 +++++++++++ 11 files changed, 335 insertions(+), 22 deletions(-) create mode 100644 .github/workflows/ci.yml create mode 100644 tests/test_price_change.py create mode 100644 tests/test_seller_info.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..3f914a9 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,35 @@ +name: Cross-Platform CI (Linux, macOS, Windows) + +on: + push: + branches: [ "all-fixes" ] + pull_request: + workflow_dispatch: + +jobs: + test: + name: ${{ matrix.os }} - Python ${{ matrix.python-version }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + python-version: ["3.11", "3.12"] + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt pytest + + - name: Run pytest + run: | + pytest -v diff --git a/db_service.py b/db_service.py index af88b6f..2be1c1b 100644 --- a/db_service.py +++ b/db_service.py @@ -25,13 +25,39 @@ def _create_table(self): cursor.execute( """ CREATE TABLE IF NOT EXISTS viewed ( - id INTEGER, + id INTEGER PRIMARY KEY, price INTEGER ) """ ) + try: + cursor.execute( + """ + DELETE FROM viewed WHERE rowid NOT IN ( + SELECT max(rowid) FROM viewed GROUP BY id + ) + """ + ) + cursor.execute( + """ + CREATE UNIQUE INDEX IF NOT EXISTS idx_viewed_id ON viewed (id) + """ + ) + except Exception: + pass conn.commit() + def get_record_price(self, record_id: int) -> int | None: + """Возвращает сохраненную цену для объявления или None, если запись отсутствует.""" + with sqlite3.connect(self.db_name) as conn: + cursor = conn.cursor() + cursor.execute( + "SELECT price FROM viewed WHERE id = ?", + (record_id,), + ) + row = cursor.fetchone() + return row[0] if row else None + def add_record(self, ad: Item): """Добавляет новую запись в таблицу viewed.""" price = ( @@ -42,7 +68,7 @@ def add_record(self, ad: Item): with sqlite3.connect(self.db_name) as conn: cursor = conn.cursor() cursor.execute( - "INSERT INTO viewed (id, price) VALUES (?, ?)", + "INSERT OR REPLACE INTO viewed (id, price) VALUES (?, ?)", (ad.id, price), ) conn.commit() diff --git a/integrations/notifications/base.py b/integrations/notifications/base.py index 2aa6692..dbac1a2 100644 --- a/integrations/notifications/base.py +++ b/integrations/notifications/base.py @@ -20,12 +20,24 @@ def notify_many(self, ads: list[Item]): def format(self, ad: Item) -> str: price = escape_markdown_v2(get_price(ad)) title = escape_markdown_v2(getattr(ad, "title", "")) - seller = escape_markdown_v2(str(getattr(ad, "sellerId", ""))) + seller_display = getattr(ad, "sellerName", None) or getattr(ad, "sellerId", "") + seller = escape_markdown_v2(str(seller_display)) if seller_display else "" short_url = f"https://avito.ru/{getattr(ad, 'id', '')}" parts = [] - if price: + if getattr(ad, "old_price", None) is not None and getattr(ad, "priceDetailed", None) and getattr(ad.priceDetailed, "value", None) is not None: + new_val = ad.priceDetailed.value + old_val = ad.old_price + old_formatted = f"{old_val:,}".replace(",", " ") + " ₽" + new_formatted = f"{new_val:,}".replace(",", " ") + " ₽" + arrow = "📉" if new_val < old_val else "📈" + price_change_text = escape_markdown_v2(f"{arrow} {old_formatted} ➔ {new_formatted}") + part = f"*{price_change_text}*" + if getattr(ad, "isPromotion", False): + part += " 🢁" + parts.append(part) + elif price: part = f"*{price}*" if getattr(ad, "isPromotion", False): part += " 🢁" diff --git a/integrations/notifications/vk.py b/integrations/notifications/vk.py index aa1c90c..76746f3 100644 --- a/integrations/notifications/vk.py +++ b/integrations/notifications/vk.py @@ -153,10 +153,21 @@ def clean(text: str) -> str: ""))) title = clean(getattr(ad, "title", "")) short_url = f"https://avito.ru/{getattr(ad, 'id', '')}" - seller = clean(str(getattr(ad, "sellerId", ""))) if getattr(ad, "sellerId", None) else "" + seller_display = getattr(ad, "sellerName", None) or getattr(ad, "sellerId", "") + seller = clean(str(seller_display)) if seller_display else "" parts = [] - if price: + if getattr(ad, "old_price", None) is not None and getattr(ad, "priceDetailed", None) and getattr(ad.priceDetailed, "value", None) is not None: + new_val = ad.priceDetailed.value + old_val = ad.old_price + old_formatted = f"{old_val:,}".replace(",", " ") + " ₽" + new_formatted = f"{new_val:,}".replace(",", " ") + " ₽" + arrow = "📉" if new_val < old_val else "📈" + price_part = f"{arrow} {old_formatted} ➔ {new_formatted}" + if getattr(ad, "isPromotion", False): + price_part += " 🔥" + parts.append(price_part) + elif price: price_part = f"💰 {price}" if getattr(ad, "isPromotion", False): price_part += " 🔥" diff --git a/load_config.py b/load_config.py index 18aaf01..3f50a75 100644 --- a/load_config.py +++ b/load_config.py @@ -1,4 +1,7 @@ -import tomllib +try: + import tomllib +except ModuleNotFoundError: + import tomli as tomllib from dataclasses import fields from pathlib import Path diff --git a/models.py b/models.py index 6b7e0d3..3bd1035 100644 --- a/models.py +++ b/models.py @@ -24,17 +24,17 @@ class AddressDetailed(BaseModel): class PriceDetailed(BaseModel): - enabled: bool - fullString: str - hasValue: bool - postfix: str - string: str - stringWithoutDiscount: Optional[str] - title: Dict[str, str] - titleDative: str - value: int - wasLowered: bool - exponent: str + enabled: bool = True + fullString: str = "" + hasValue: bool = True + postfix: str = "₽" + string: str = "" + stringWithoutDiscount: Optional[str] = None + title: Dict[str, str] = {} + titleDative: str = "" + value: int = 0 + wasLowered: bool = False + exponent: str = "" class Image(RootModel): @@ -83,7 +83,7 @@ class Gallery(BaseModel): class UserLogo(BaseModel): link: str | None = None src: HttpUrl | str | None = None - developerId: Optional[int] + developerId: Optional[int] = None class IvaComponent(BaseModel): @@ -144,6 +144,8 @@ class Item(BaseModel): closestAddressId: int | None = None isSparePartsCompatibility: bool | None = None sellerId: str | None = None + sellerName: str | None = None + old_price: int | None = None isPromotion: bool = False total_views: int | None = None today_views: int | None = None diff --git a/parser/export/excel.py b/parser/export/excel.py index c90513d..4a4f129 100644 --- a/parser/export/excel.py +++ b/parser/export/excel.py @@ -127,13 +127,17 @@ def save(self, ads: list[Item]) -> None: url_path = f"/{url_path}" item_url = f"https://www.avito.ru{url_path}" if url_path else "" + seller_val = getattr(ad, "sellerName", None) or getattr(ad, "sellerId", "") + if getattr(ad, "sellerName", None) and getattr(ad, "sellerId", None) and ad.sellerName != ad.sellerId: + seller_val = f"{ad.sellerName} ({ad.sellerId})" + row = [ self.excel_safe(ad.title), price_value, self.excel_safe(item_url), self.excel_safe(ad.description), self._get_ad_time(ad), - self.excel_safe(ad.sellerId or ""), + self.excel_safe(seller_val), self.excel_safe(ad.location.name if ad.location else ""), self.excel_safe(self._get_item_address_user(ad)), self.excel_safe(self._get_item_coords(ad)), diff --git a/parser_cls.py b/parser_cls.py index a27c863..2736fab 100644 --- a/parser_cls.py +++ b/parser_cls.py @@ -441,18 +441,56 @@ def _extract_description(html: str) -> str | None: @staticmethod def _extract_seller_slug(data): - match = re.search(r"/brands/([^/?#]+)", str(data)) + """Извлекает идентификатор/slug продавца из объекта объявления (бренды или профиль пользователя).""" + user_logo = getattr(data, "userLogo", None) + if user_logo and getattr(user_logo, "link", None): + match = re.search(r"/(?:brands|user)/([^/?#'\"\s]+)", str(user_logo.link)) + if match: + return match.group(1).rstrip("/") + + match = re.search(r"/(?:brands|user)/([a-zA-Z0-9_\-]+)", str(data)) if match: return match.group(1) + + return None + + @staticmethod + def _extract_seller_name(html: str) -> str | None: + """Извлекает имя продавца со страницы объявления (Issue #333).""" + soup = BeautifulSoup(html, "html.parser") + for marker in ['[data-marker="seller-info/name"]', '[data-marker="seller-link/link"]', '[data-marker="seller-info/label"]']: + el = soup.select_one(marker) + if el and el.get_text(strip=True): + return el.get_text(strip=True) + + try: + import html as html_lib + for script in soup.select('script[type="mime/invalid"][data-mfe-state="true"]'): + if 'sandbox' not in script.text: + data = json.loads(html_lib.unescape(script.text)) + item_data = data.get('loaderData', {}).get("data", {}).get("item", {}) + seller = item_data.get("seller", {}) + if name := seller.get("name"): + return name.strip() + except Exception: + pass + return None def is_viewed(self, ad: Item) -> bool: - """Проверяет, смотрели мы это или нет""" + """Проверяет, смотрели мы это или нет. Если цена изменилась, фиксирует старую цену (Issue #214).""" price = ( ad.priceDetailed.value if (ad.priceDetailed and getattr(ad.priceDetailed, "value", None) is not None) else 0 ) + if hasattr(self.db_handler, "get_record_price"): + saved_price = self.db_handler.get_record_price(ad.id) + if isinstance(saved_price, int): + if saved_price != price: + ad.old_price = saved_price + return False + return True return self.db_handler.record_exists(record_id=ad.id, price=price) @staticmethod diff --git a/tests/test_excel_export.py b/tests/test_excel_export.py index e5bd53e..c6787ca 100644 --- a/tests/test_excel_export.py +++ b/tests/test_excel_export.py @@ -124,3 +124,23 @@ def test_excel_coords_handling(): assert row1[8] == "55.75;37.61" assert row2[8] in ("", None) assert row3[8] in ("", None) + + +def test_excel_seller_name_and_id(): + with tempfile.TemporaryDirectory() as tmp_dir: + file_path = Path(tmp_dir) / "seller_export.xlsx" + excel = Excel(file_path=file_path) + + ad_both = Item(id=1, sellerName="Мебель Люкс", sellerId="mebel_shop") + ad_name_only = Item(id=2, sellerName="Алексей Частник", sellerId=None) + ad_id_only = Item(id=3, sellerName=None, sellerId="shop123") + + excel.save([ad_both, ad_name_only, ad_id_only]) + + wb = load_workbook(file_path) + sheet = wb.active + # Col index 5 is seller (0-indexed: Заголовок, Цена, URL, Описание, Дата, Продавец) + assert sheet.cell(row=2, column=6).value == "Мебель Люкс (mebel_shop)" + assert sheet.cell(row=3, column=6).value == "Алексей Частник" + assert sheet.cell(row=4, column=6).value == "shop123" + diff --git a/tests/test_price_change.py b/tests/test_price_change.py new file mode 100644 index 0000000..128b5d0 --- /dev/null +++ b/tests/test_price_change.py @@ -0,0 +1,118 @@ +import sqlite3 +from unittest.mock import MagicMock +from models import Item, PriceDetailed +from db_service import SQLiteDBHandler +from integrations.notifications.base import Notifier + + +class DummyNotifier(Notifier): + def notify(self, ad: Item = None, message: str = None): + return self.format(ad) if ad else message + + +def test_db_get_record_price(tmp_path): + db_file = str(tmp_path / "test.db") + db = SQLiteDBHandler.__new__(SQLiteDBHandler) + db.db_name = db_file + db._create_table() + + # Initially None + assert db.get_record_price(100) is None + + # Insert record with price 1000 + item = Item(id=100, priceDetailed=PriceDetailed( + enabled=True, fullString="1 000 ₽", hasValue=True, postfix="₽", + string="1 000 ₽", title={}, titleDative="", value=1000, + wasLowered=False, exponent="" + )) + db.add_record_from_page([item]) + + assert db.get_record_price(100) == 1000 + + # Update with new price 800 + item_updated = Item(id=100, priceDetailed=PriceDetailed( + enabled=True, fullString="800 ₽", hasValue=True, postfix="₽", + string="800 ₽", title={}, titleDative="", value=800, + wasLowered=True, exponent="" + )) + db.add_record_from_page([item_updated]) + + assert db.get_record_price(100) == 800 + + +def test_price_change_notification_format(): + notifier = DummyNotifier() + + # Ad with price dropped + ad_lowered = Item( + id=123, + title="Тестовый товар", + priceDetailed=PriceDetailed( + enabled=True, fullString="8 500 ₽", hasValue=True, postfix="₽", + string="8 500 ₽", title={}, titleDative="", value=8500, + wasLowered=True, exponent="" + ), + old_price=10000 + ) + + formatted = notifier.format(ad_lowered) + assert "8 500" in formatted + assert "10 000" in formatted + assert "📉" in formatted or "Цена" in formatted + + # Ad with price raised + ad_raised = Item( + id=124, + title="Тестовый товар 2", + priceDetailed=PriceDetailed( + enabled=True, fullString="12 000 ₽", hasValue=True, postfix="₽", + string="12 000 ₽", title={}, titleDative="", value=12000, + wasLowered=False, exponent="" + ), + old_price=10000 + ) + + formatted_raised = notifier.format(ad_raised) + assert "12 000" in formatted_raised + assert "10 000" in formatted_raised + assert "📈" in formatted_raised or "Цена" in formatted_raised + + +def test_new_ad_notification_format_without_old_price(): + notifier = DummyNotifier() + + ad_new = Item( + id=125, + title="Новый товар", + priceDetailed=PriceDetailed( + enabled=True, fullString="5 000 ₽", hasValue=True, postfix="₽", + string="5 000 ₽", title={}, titleDative="", value=5000, + wasLowered=False, exponent="" + ), + old_price=None + ) + + formatted_new = notifier.format(ad_new) + assert "5000" in formatted_new + assert "➔" not in formatted_new + + +def test_vk_notification_price_change(): + from integrations.notifications.vk import VKNotifier + ad_lowered = Item( + id=126, + title="Ноутбук", + priceDetailed=PriceDetailed( + enabled=True, fullString="45 000 ₽", hasValue=True, postfix="₽", + string="45 000 ₽", title={}, titleDative="", value=45000, + wasLowered=True, exponent="" + ), + old_price=50000, + sellerName="Иван Продавец" + ) + + formatted = VKNotifier.format_ad(ad_lowered) + assert "45 000" in formatted + assert "50 000" in formatted + assert "📉" in formatted + assert "Иван Продавец" in formatted diff --git a/tests/test_seller_info.py b/tests/test_seller_info.py new file mode 100644 index 0000000..62d1ae7 --- /dev/null +++ b/tests/test_seller_info.py @@ -0,0 +1,44 @@ +from bs4 import BeautifulSoup +from models import Item, UserLogo +from parser_cls import AvitoParse + + +def test_extract_seller_slug_brands(): + ad = Item(id=1, userLogo=UserLogo(link="/brands/techstore")) + slug = AvitoParse._extract_seller_slug(ad) + assert slug == "techstore" + + +def test_extract_seller_slug_user(): + ad = Item(id=2, userLogo=UserLogo(link="/user/a1b2c3d4e5f/profile")) + slug = AvitoParse._extract_seller_slug(ad) + assert slug == "a1b2c3d4e5f" + + +def test_extract_seller_name_from_html_marker(): + html = """ + + +
+ Алексей (частное лицо) +
+ + + """ + name = AvitoParse._extract_seller_name(html) + assert name == "Алексей (частное лицо)" + + +def test_extract_seller_name_from_json(): + html = """ + + + + + + + """ + name = AvitoParse._extract_seller_name(html) + assert name == "ООО Мебель-Люкс"