Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 35 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
name: Cross-Platform CI (Linux, macOS, Windows)

on:
push:
branches: [ "all-fixes" ]
pull_request:
workflow_dispatch:

jobs:
test:
name: ${{ matrix.os }} - Python ${{ matrix.python-version }}
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
matrix:
os: [ubuntu-latest, macos-latest, windows-latest]
python-version: ["3.11", "3.12"]

steps:
- name: Checkout repository
uses: actions/checkout@v4

- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}

- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install -r requirements.txt pytest

- name: Run pytest
run: |
pytest -v
30 changes: 28 additions & 2 deletions db_service.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,13 +25,39 @@ def _create_table(self):
cursor.execute(
"""
CREATE TABLE IF NOT EXISTS viewed (
id INTEGER,
id INTEGER PRIMARY KEY,
price INTEGER
)
"""
)
try:
cursor.execute(
"""
DELETE FROM viewed WHERE rowid NOT IN (
SELECT max(rowid) FROM viewed GROUP BY id
)
"""
)
cursor.execute(
"""
CREATE UNIQUE INDEX IF NOT EXISTS idx_viewed_id ON viewed (id)
"""
)
except Exception:
pass
conn.commit()

def get_record_price(self, record_id: int) -> int | None:
"""Возвращает сохраненную цену для объявления или None, если запись отсутствует."""
with sqlite3.connect(self.db_name) as conn:
cursor = conn.cursor()
cursor.execute(
"SELECT price FROM viewed WHERE id = ?",
(record_id,),
)
row = cursor.fetchone()
return row[0] if row else None

def add_record(self, ad: Item):
"""Добавляет новую запись в таблицу viewed."""
price = (
Expand All @@ -42,7 +68,7 @@ def add_record(self, ad: Item):
with sqlite3.connect(self.db_name) as conn:
cursor = conn.cursor()
cursor.execute(
"INSERT INTO viewed (id, price) VALUES (?, ?)",
"INSERT OR REPLACE INTO viewed (id, price) VALUES (?, ?)",
(ad.id, price),
)
conn.commit()
Expand Down
16 changes: 14 additions & 2 deletions integrations/notifications/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,12 +20,24 @@ def notify_many(self, ads: list[Item]):
def format(self, ad: Item) -> str:
price = escape_markdown_v2(get_price(ad))
title = escape_markdown_v2(getattr(ad, "title", ""))
seller = escape_markdown_v2(str(getattr(ad, "sellerId", "")))
seller_display = getattr(ad, "sellerName", None) or getattr(ad, "sellerId", "")
seller = escape_markdown_v2(str(seller_display)) if seller_display else ""
short_url = f"https://avito.ru/{getattr(ad, 'id', '')}"

parts = []

if price:
if getattr(ad, "old_price", None) is not None and getattr(ad, "priceDetailed", None) and getattr(ad.priceDetailed, "value", None) is not None:
new_val = ad.priceDetailed.value
old_val = ad.old_price
old_formatted = f"{old_val:,}".replace(",", " ") + " ₽"
new_formatted = f"{new_val:,}".replace(",", " ") + " ₽"
arrow = "📉" if new_val < old_val else "📈"
price_change_text = escape_markdown_v2(f"{arrow} {old_formatted} ➔ {new_formatted}")
part = f"*{price_change_text}*"
if getattr(ad, "isPromotion", False):
part += " 🢁"
parts.append(part)
elif price:
part = f"*{price}*"
if getattr(ad, "isPromotion", False):
part += " 🢁"
Expand Down
15 changes: 13 additions & 2 deletions integrations/notifications/vk.py
Original file line number Diff line number Diff line change
Expand Up @@ -153,10 +153,21 @@ def clean(text: str) -> str:
"")))
title = clean(getattr(ad, "title", ""))
short_url = f"https://avito.ru/{getattr(ad, 'id', '')}"
seller = clean(str(getattr(ad, "sellerId", ""))) if getattr(ad, "sellerId", None) else ""
seller_display = getattr(ad, "sellerName", None) or getattr(ad, "sellerId", "")
seller = clean(str(seller_display)) if seller_display else ""

parts = []
if price:
if getattr(ad, "old_price", None) is not None and getattr(ad, "priceDetailed", None) and getattr(ad.priceDetailed, "value", None) is not None:
new_val = ad.priceDetailed.value
old_val = ad.old_price
old_formatted = f"{old_val:,}".replace(",", " ") + " ₽"
new_formatted = f"{new_val:,}".replace(",", " ") + " ₽"
arrow = "📉" if new_val < old_val else "📈"
price_part = f"{arrow} {old_formatted} ➔ {new_formatted}"
if getattr(ad, "isPromotion", False):
price_part += " 🔥"
parts.append(price_part)
elif price:
price_part = f"💰 {price}"
if getattr(ad, "isPromotion", False):
price_part += " 🔥"
Expand Down
5 changes: 4 additions & 1 deletion load_config.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,7 @@
import tomllib
try:
import tomllib
except ModuleNotFoundError:
import tomli as tomllib
from dataclasses import fields
from pathlib import Path

Expand Down
26 changes: 14 additions & 12 deletions models.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,17 +24,17 @@ class AddressDetailed(BaseModel):


class PriceDetailed(BaseModel):
enabled: bool
fullString: str
hasValue: bool
postfix: str
string: str
stringWithoutDiscount: Optional[str]
title: Dict[str, str]
titleDative: str
value: int
wasLowered: bool
exponent: str
enabled: bool = True
fullString: str = ""
hasValue: bool = True
postfix: str = "₽"
string: str = ""
stringWithoutDiscount: Optional[str] = None
title: Dict[str, str] = {}
titleDative: str = ""
value: int = 0
wasLowered: bool = False
exponent: str = ""


class Image(RootModel):
Expand Down Expand Up @@ -83,7 +83,7 @@ class Gallery(BaseModel):
class UserLogo(BaseModel):
link: str | None = None
src: HttpUrl | str | None = None
developerId: Optional[int]
developerId: Optional[int] = None


class IvaComponent(BaseModel):
Expand Down Expand Up @@ -144,6 +144,8 @@ class Item(BaseModel):
closestAddressId: int | None = None
isSparePartsCompatibility: bool | None = None
sellerId: str | None = None
sellerName: str | None = None
old_price: int | None = None
isPromotion: bool = False
total_views: int | None = None
today_views: int | None = None
Expand Down
6 changes: 5 additions & 1 deletion parser/export/excel.py
Original file line number Diff line number Diff line change
Expand Up @@ -127,13 +127,17 @@ def save(self, ads: list[Item]) -> None:
url_path = f"/{url_path}"
item_url = f"https://www.avito.ru{url_path}" if url_path else ""

seller_val = getattr(ad, "sellerName", None) or getattr(ad, "sellerId", "")
if getattr(ad, "sellerName", None) and getattr(ad, "sellerId", None) and ad.sellerName != ad.sellerId:
seller_val = f"{ad.sellerName} ({ad.sellerId})"

row = [
self.excel_safe(ad.title),
price_value,
self.excel_safe(item_url),
self.excel_safe(ad.description),
self._get_ad_time(ad),
self.excel_safe(ad.sellerId or ""),
self.excel_safe(seller_val),
self.excel_safe(ad.location.name if ad.location else ""),
self.excel_safe(self._get_item_address_user(ad)),
self.excel_safe(self._get_item_coords(ad)),
Expand Down
42 changes: 40 additions & 2 deletions parser_cls.py
Original file line number Diff line number Diff line change
Expand Up @@ -441,18 +441,56 @@ def _extract_description(html: str) -> str | None:

@staticmethod
def _extract_seller_slug(data):
match = re.search(r"/brands/([^/?#]+)", str(data))
"""Извлекает идентификатор/slug продавца из объекта объявления (бренды или профиль пользователя)."""
user_logo = getattr(data, "userLogo", None)
if user_logo and getattr(user_logo, "link", None):
match = re.search(r"/(?:brands|user)/([^/?#'\"\s]+)", str(user_logo.link))
if match:
return match.group(1).rstrip("/")

match = re.search(r"/(?:brands|user)/([a-zA-Z0-9_\-]+)", str(data))
if match:
return match.group(1)

return None

@staticmethod
def _extract_seller_name(html: str) -> str | None:
"""Извлекает имя продавца со страницы объявления (Issue #333)."""
soup = BeautifulSoup(html, "html.parser")
for marker in ['[data-marker="seller-info/name"]', '[data-marker="seller-link/link"]', '[data-marker="seller-info/label"]']:
el = soup.select_one(marker)
if el and el.get_text(strip=True):
return el.get_text(strip=True)

try:
import html as html_lib
for script in soup.select('script[type="mime/invalid"][data-mfe-state="true"]'):
if 'sandbox' not in script.text:
data = json.loads(html_lib.unescape(script.text))
item_data = data.get('loaderData', {}).get("data", {}).get("item", {})
seller = item_data.get("seller", {})
if name := seller.get("name"):
return name.strip()
except Exception:
pass

return None

def is_viewed(self, ad: Item) -> bool:
"""Проверяет, смотрели мы это или нет"""
"""Проверяет, смотрели мы это или нет. Если цена изменилась, фиксирует старую цену (Issue #214)."""
price = (
ad.priceDetailed.value
if (ad.priceDetailed and getattr(ad.priceDetailed, "value", None) is not None)
else 0
)
if hasattr(self.db_handler, "get_record_price"):
saved_price = self.db_handler.get_record_price(ad.id)
if isinstance(saved_price, int):
if saved_price != price:
ad.old_price = saved_price
return False
return True
return self.db_handler.record_exists(record_id=ad.id, price=price)

@staticmethod
Expand Down
20 changes: 20 additions & 0 deletions tests/test_excel_export.py
Original file line number Diff line number Diff line change
Expand Up @@ -124,3 +124,23 @@ def test_excel_coords_handling():
assert row1[8] == "55.75;37.61"
assert row2[8] in ("", None)
assert row3[8] in ("", None)


def test_excel_seller_name_and_id():
with tempfile.TemporaryDirectory() as tmp_dir:
file_path = Path(tmp_dir) / "seller_export.xlsx"
excel = Excel(file_path=file_path)

ad_both = Item(id=1, sellerName="Мебель Люкс", sellerId="mebel_shop")
ad_name_only = Item(id=2, sellerName="Алексей Частник", sellerId=None)
ad_id_only = Item(id=3, sellerName=None, sellerId="shop123")

excel.save([ad_both, ad_name_only, ad_id_only])

wb = load_workbook(file_path)
sheet = wb.active
# Col index 5 is seller (0-indexed: Заголовок, Цена, URL, Описание, Дата, Продавец)
assert sheet.cell(row=2, column=6).value == "Мебель Люкс (mebel_shop)"
assert sheet.cell(row=3, column=6).value == "Алексей Частник"
assert sheet.cell(row=4, column=6).value == "shop123"

Loading
Loading