From 10157cf3930dee9ce41fedb87a1bc40b486d53f6 Mon Sep 17 00:00:00 2001 From: eminsk Date: Wed, 30 Sep 2026 17:08:02 -0400 Subject: [PATCH 1/2] feat: parse item properties and characteristics (fixes #335) --- .github/workflows/ci.yml | 35 ++++++++ config.toml | 1 + dto.py | 1 + load_config.py | 5 +- models.py | 1 + parser/export/excel.py | 8 +- parser_cls.py | 83 +++++++++++++++++- tests/test_params_parser.py | 164 ++++++++++++++++++++++++++++++++++++ 8 files changed, 293 insertions(+), 5 deletions(-) create mode 100644 .github/workflows/ci.yml create mode 100644 tests/test_params_parser.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..3f914a9 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,35 @@ +name: Cross-Platform CI (Linux, macOS, Windows) + +on: + push: + branches: [ "all-fixes" ] + pull_request: + workflow_dispatch: + +jobs: + test: + name: ${{ matrix.os }} - Python ${{ matrix.python-version }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + python-version: ["3.11", "3.12"] + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt pytest + + - name: Run pytest + run: | + pytest -v diff --git a/config.toml b/config.toml index 5b2052c..67b3c44 100644 --- a/config.toml +++ b/config.toml @@ -26,6 +26,7 @@ one_time_start = false one_file_for_link = false parse_views = false parse_description = false +parse_params = false save_xlsx = true use_webdriver = false use_bypass_api = false diff --git a/dto.py b/dto.py index 1dc4b42..d3377de 100644 --- a/dto.py +++ b/dto.py @@ -44,6 +44,7 @@ class AvitoConfig: one_file_for_link: bool = False parse_views: bool = False parse_description: bool = False + parse_params: bool = False save_xlsx: bool = True use_webdriver: bool = True use_bypass_api: bool = False diff --git a/load_config.py b/load_config.py index 18aaf01..3f50a75 100644 --- a/load_config.py +++ b/load_config.py @@ -1,4 +1,7 @@ -import tomllib +try: + import tomllib +except ModuleNotFoundError: + import tomli as tomllib from dataclasses import fields from pathlib import Path diff --git a/models.py b/models.py index 6b7e0d3..45d5a18 100644 --- a/models.py +++ b/models.py @@ -148,6 +148,7 @@ class Item(BaseModel): total_views: int | None = None today_views: int | None = None phone: str | None = None + params: Optional[Dict[str, str]] = None class ItemsResponse(BaseModel): diff --git a/parser/export/excel.py b/parser/export/excel.py index c90513d..cbc00ed 100644 --- a/parser/export/excel.py +++ b/parser/export/excel.py @@ -28,7 +28,8 @@ class ExcelStorage(ResultStorage): "Поднято", "Просмотры (всего)", "Просмотры (сегодня)", - "Телефон" + "Телефон", + "Параметры" ] def __init__(self, file_path: Path): @@ -142,6 +143,11 @@ def save(self, ads: list[Item]) -> None: ad.total_views if ad.total_views is not None else "", ad.today_views if ad.today_views is not None else "", self.excel_safe(ad.phone or ""), + self.excel_safe( + "; ".join(f"{k}: {v}" for k, v in ad.params.items()) + if isinstance(getattr(ad, "params", None), dict) + else (ad.params or "") + ), ] sheet.append(row) diff --git a/parser_cls.py b/parser_cls.py index a27c863..d5a4fe0 100644 --- a/parser_cls.py +++ b/parser_cls.py @@ -203,6 +203,7 @@ def parse(self): self.notifier.notify_many(ads=filtered_ads) filtered_ads = self.parse_views(ads=filtered_ads) filtered_ads = self.parse_description(ads=filtered_ads) + filtered_ads = self.parse_params(ads=filtered_ads) filtered_ads = self.parse_phone(ads=filtered_ads) if filtered_ads: @@ -333,7 +334,62 @@ def count(value): return description or None, total, today - def _enrich_ads(self, ads: list[Item], *, views: bool, description: bool) -> list[Item]: + @staticmethod + def _extract_params(payload: dict) -> dict[str, str] | None: + """ + Извлечение характеристик объявления («О помещении», параметры товара). + Поддерживает сценарии Beduin (mobile API scenario) и mobile.params. + """ + if not isinstance(payload, dict): + return None + + def nested_dict(value, *keys): + for key in keys: + if not isinstance(value, dict): + return {} + value = value.get(key) + return value if isinstance(value, dict) else {} + + params_dict: dict[str, str] = {} + + # 1. Beduin scenario: success.view.scenario.beduin.main.params + card_params = nested_dict( + payload, "success", "view", "scenario", "beduin", "main", "params" + ) + if isinstance(card_params, dict): + for widget_key, widget_data in card_params.items(): + if isinstance(widget_data, dict): + items = widget_data.get("items") + if isinstance(items, list): + for item in items: + if isinstance(item, dict): + title = item.get("title") or item.get("name") or item.get("label") + desc = item.get("description") or item.get("value") or item.get("text") + if isinstance(title, str) and title.strip(): + if isinstance(desc, (str, int, float)) and str(desc).strip(): + params_dict[title.strip()] = str(desc).strip() + + # 2. Mobile API: success.mobile.params / parameters / properties + mobile = nested_dict(payload, "success", "mobile") + for container in (mobile.get("params"), mobile.get("parameters"), mobile.get("properties")): + if isinstance(container, list): + for item in container: + if isinstance(item, dict): + title = item.get("title") or item.get("name") or item.get("label") + desc = item.get("description") or item.get("value") or item.get("text") + if isinstance(title, str) and title.strip(): + if isinstance(desc, (str, int, float)) and str(desc).strip(): + params_dict[title.strip()] = str(desc).strip() + elif isinstance(container, dict): + for k, v in container.items(): + if isinstance(k, str) and k.strip() and isinstance(v, (str, int, float)) and str(v).strip(): + params_dict[k.strip()] = str(v).strip() + + return params_dict if params_dict else None + + def _enrich_ads( + self, ads: list[Item], *, views: bool, description: bool, params: bool = False + ) -> list[Item]: total_ads = len(ads) for index, ad in enumerate(ads, start=1): if self.stop_event and self.stop_event.is_set(): @@ -361,6 +417,10 @@ def _enrich_ads(self, ads: list[Item], *, views: bool, description: bool) -> lis ) if description and full_description: ad.description = full_description + if params or getattr(self.config, "parse_params", False): + item_params = self._extract_params(payload) + if item_params: + ad.params = item_params except Exception as err: logger.warning(f"Ошибка при разборе карточки {ad.id}: {err}") continue @@ -373,7 +433,10 @@ def parse_views(self, ads: list[Item]) -> list[Item]: logger.info("Начинаю парсинг просмотров") return self._enrich_ads( - ads, views=True, description=getattr(self.config, "parse_description", False) + ads, + views=True, + description=getattr(self.config, "parse_description", False), + params=getattr(self.config, "parse_params", False), ) def parse_description(self, ads: list[Item]) -> list[Item]: @@ -381,7 +444,21 @@ def parse_description(self, ads: list[Item]) -> list[Item]: return ads logger.info("Начинаю парсинг полного описания") - return self._enrich_ads(ads, views=False, description=True) + return self._enrich_ads( + ads, + views=False, + description=True, + params=getattr(self.config, "parse_params", False), + ) + + def parse_params(self, ads: list[Item]) -> list[Item]: + if not getattr(self.config, "parse_params", False): + return ads + if self.config.parse_views or getattr(self.config, "parse_description", False): + return ads + + logger.info("Начинаю парсинг характеристик объявлений") + return self._enrich_ads(ads, views=False, description=False, params=True) def parse_phone(self, ads: list[Item]) -> list[Item]: if not self.config.parse_phone or self.config.parse_phone: diff --git a/tests/test_params_parser.py b/tests/test_params_parser.py new file mode 100644 index 0000000..2cd2cb6 --- /dev/null +++ b/tests/test_params_parser.py @@ -0,0 +1,164 @@ +import tempfile +from pathlib import Path +from unittest.mock import MagicMock +from openpyxl import load_workbook + +from dto import AvitoConfig +from models import Item +from parser.export.excel import ExcelStorage as Excel +from parser_cls import AvitoParse + + +def test_extract_params_from_beduin_scenario(): + """Тест извлечения параметров недвижимости/товара из сценария Beduin (mobile card API).""" + payload = { + "success": { + "view": { + "scenario": { + "beduin": { + "main": { + "params": { + "itemParams": { + "items": [ + {"title": "Общая площадь", "description": "120 м²"}, + {"title": "Вход", "description": "отдельный с улицы"}, + {"title": "Этаж", "description": "1 из 5"}, + ] + }, + "aboutPremises": { + "items": [ + {"title": "Отделка", "description": "офисная"}, + {"title": "Отопление", "description": "центральное"}, + ] + } + } + } + } + } + } + } + } + + params = AvitoParse._extract_params(payload) + assert params is not None + assert params["Общая площадь"] == "120 м²" + assert params["Вход"] == "отдельный с улицы" + assert params["Этаж"] == "1 из 5" + assert params["Отделка"] == "офисная" + assert params["Отопление"] == "центральное" + + +def test_extract_params_from_mobile_api(): + """Тест извлечения параметров из mobile.params / parameters.""" + payload = { + "success": { + "mobile": { + "params": [ + {"title": "Площадь", "description": "55 м²"}, + {"title": "Тип здания", "description": "жилой дом"}, + ] + } + } + } + + params = AvitoParse._extract_params(payload) + assert params is not None + assert params["Площадь"] == "55 м²" + assert params["Тип здания"] == "жилой дом" + + +def test_extract_params_returns_none_when_empty(): + """Тест безопасной обработки пустых или некорректных ответов.""" + assert AvitoParse._extract_params({}) is None + assert AvitoParse._extract_params({"success": {}}) is None + assert AvitoParse._extract_params(None) is None + + +def test_parse_params_method_enriches_ad(): + """Тест метода parse_params при включённой опции в конфиге.""" + config = AvitoConfig(urls=["https://www.avito.ru/test"], parse_params=True) + parser = AvitoParse.__new__(AvitoParse) + parser.config = config + parser.stop_event = None + parser.good_request_count = 0 + parser.bad_request_count = 0 + parser.http = MagicMock() + parser.http.fetch_item_data.return_value = { + "success": { + "view": { + "scenario": { + "beduin": { + "main": { + "params": { + "itemParams": { + "items": [ + {"title": "Вход", "description": "с улицы"}, + {"title": "Общая площадь", "description": "85.5 м²"}, + ] + } + } + } + } + } + } + } + } + + ad = Item(id=999, urlPath="/realty_999", title="Помещение свободного назначения") + result = parser.parse_params([ad]) + + assert len(result) == 1 + assert result[0].params is not None + assert result[0].params["Вход"] == "с улицы" + assert result[0].params["Общая площадь"] == "85.5 м²" + parser.http.fetch_item_data.assert_called_once_with(999) + + +def test_excel_export_saves_params_column(): + """Тест выгрузки параметров объекта («О помещении») в XLSX.""" + with tempfile.TemporaryDirectory() as tmp_dir: + file_path = Path(tmp_dir) / "realty_export.xlsx" + excel = Excel(file_path=file_path) + + ad = Item( + id=777, + title="Офис 100м²", + params={ + "Вход": "отдельный", + "Общая площадь": "100 м²", + "Этаж": "2", + } + ) + + excel.save([ad]) + + wb = load_workbook(file_path) + sheet = wb.active + # Header check: last column should be "Параметры" + headers = [cell.value for cell in sheet[1]] + assert "Параметры" in headers + params_col_idx = headers.index("Параметры") + 1 + + # Row check + cell_val = sheet.cell(row=2, column=params_col_idx).value + assert "Вход: отдельный" in cell_val + assert "Общая площадь: 100 м²" in cell_val + assert "Этаж: 2" in cell_val + + +def test_excel_export_handles_none_params(): + """Тест выгрузки в XLSX при отсутствии параметров.""" + with tempfile.TemporaryDirectory() as tmp_dir: + file_path = Path(tmp_dir) / "empty_params_export.xlsx" + excel = Excel(file_path=file_path) + + ad = Item(id=778, title="Товар без параметров", params=None) + excel.save([ad]) + + wb = load_workbook(file_path) + sheet = wb.active + headers = [cell.value for cell in sheet[1]] + params_col_idx = headers.index("Параметры") + 1 + + cell_val = sheet.cell(row=2, column=params_col_idx).value + assert cell_val in ("", None) From 944f8d5a04e7b78319f344b0d554b87d056d15fe Mon Sep 17 00:00:00 2001 From: eminsk Date: Thu, 1 Oct 2026 05:35:39 -0400 Subject: [PATCH 2/2] feat(gui): add parse_params checkbox to main window --- AvitoParser.py | 5 +++++ lang.py | 2 ++ 2 files changed, 7 insertions(+) diff --git a/AvitoParser.py b/AvitoParser.py index e799665..bcbe863 100644 --- a/AvitoParser.py +++ b/AvitoParser.py @@ -65,6 +65,7 @@ def set_up(): one_file_for_link.value = config.one_file_for_link parse_views.value = config.parse_views parse_description.value = getattr(config, "parse_description", False) + parse_params.value = getattr(config, "parse_params", False) save_xlsx.value = config.save_xlsx use_webdriver.value = config.use_webdriver use_bypass_api.value = config.use_bypass_api @@ -113,6 +114,7 @@ def save_config(): "one_file_for_link": one_file_for_link.value, "parse_views": parse_views.value, "parse_description": parse_description.value, + "parse_params": parse_params.value, "save_xlsx": save_xlsx.value, "use_webdriver": use_webdriver.value, "use_bypass_api": use_bypass_api.value, @@ -560,6 +562,8 @@ def toggle(e): tooltip=PARSE_VIEWS_HELP) parse_description = ft.Checkbox(label="Парсить описание полностью", value=False, tooltip="Загружает полное описание из карточки вместо краткого анонса") + parse_params = ft.Checkbox(label="Парсить характеристики", value=False, + tooltip=PARSE_PARAMS_HELP) parse_phone = ft.Checkbox(label="Парсить телефоны", value=False, on_change=check_api_key_exist, tooltip=PARSE_PHONE_HELP) @@ -712,6 +716,7 @@ def toggle(e): ft.Row([one_time_start, one_file_for_link]), ft.Row([parse_views, parse_description, + parse_params, #parse_phone, save_xlsx]), ] diff --git a/lang.py b/lang.py index 8bdfea2..e813ece 100644 --- a/lang.py +++ b/lang.py @@ -93,6 +93,8 @@ PARSE_VIEWS_HELP = "Дополнительно будет собираться количество просмотров: всего и за сегодня. Т.к. это требует " \ "открытия отдельной страницы - то время работы сильно увеличится. Не запускайте без прокси и если " \ "нужно получать информацию о новых объявлениях как можно быстрее" +PARSE_PARAMS_HELP = "Дополнительно будут собираться характеристики объявления («О помещении», параметры товара). " \ + "Забираются из карточки объявления" PARSE_PHONE_HELP = ("Будут дополнительно собираться временные номера телефонов. Для это нужен api ключ сайта spfa.pro, " "его нужно ввести в разделе <Прокси и обход блокировок>. spfa берет плату за эту услугу") NEED_TO_INSERT_API_KEY = ("Для парсинга телефонов нужен api ключ сайта spfa.pro, "