From 017d47248413ae6dd6b5f137c6817a3af9ca7997 Mon Sep 17 00:00:00 2001 From: krzywon Date: Thu, 17 Sep 2026 15:00:31 -0400 Subject: [PATCH 1/9] Update type hints for importers to allow pathlib.Path objects --- sasdata/data_io/importers/import_ascii.py | 3 ++- sasdata/data_io/importers/import_hdf5.py | 3 ++- sasdata/data_io/importers/import_sesans.py | 3 ++- sasdata/data_io/importers/import_xml.py | 3 ++- 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/sasdata/data_io/importers/import_ascii.py b/sasdata/data_io/importers/import_ascii.py index 938031e7..d250ab48 100644 --- a/sasdata/data_io/importers/import_ascii.py +++ b/sasdata/data_io/importers/import_ascii.py @@ -2,6 +2,7 @@ from dataclasses import dataclass, field, replace from enum import Enum from os import path +from pathlib import Path import numpy as np @@ -219,6 +220,6 @@ def load_data(params: AsciiReaderParams) -> list[SasMeasurement]: return loaded_data -def load_data_default_params(filename: str) -> list[SasMeasurement]: +def load_data_default_params(filename: str | Path) -> list[SasMeasurement]: params = guess_params_from_filename(filename, guess_dataset_type(filename)) return load_data(params) diff --git a/sasdata/data_io/importers/import_hdf5.py b/sasdata/data_io/importers/import_hdf5.py index fc883519..372ea285 100644 --- a/sasdata/data_io/importers/import_hdf5.py +++ b/sasdata/data_io/importers/import_hdf5.py @@ -1,5 +1,6 @@ import logging from collections.abc import Callable +from pathlib import Path import h5py import numpy as np @@ -404,7 +405,7 @@ def parse_metadata(node : HDF5Group) -> Metadata: raw=raw) -def load_data(filename: str) -> dict[str, SasMeasurement]: +def load_data(filename: str | Path) -> dict[str, SasMeasurement]: with h5py.File(filename, "r") as f: loaded_data: dict[str, SasMeasurement] = {} diff --git a/sasdata/data_io/importers/import_sesans.py b/sasdata/data_io/importers/import_sesans.py index 00094958..bb69e1bc 100644 --- a/sasdata/data_io/importers/import_sesans.py +++ b/sasdata/data_io/importers/import_sesans.py @@ -5,6 +5,7 @@ import re from collections import defaultdict from itertools import groupby +from pathlib import Path import numpy as np @@ -204,7 +205,7 @@ def parse_sesans(lines: list[str]) -> SasMeasurement: ) -def load_data(filename) -> SasMeasurement: +def load_data(filename: str | Path) -> SasMeasurement: with open(filename) as infile: lines = infile.readlines() return parse_sesans(lines) diff --git a/sasdata/data_io/importers/import_xml.py b/sasdata/data_io/importers/import_xml.py index 8fce64ed..3d7a329b 100644 --- a/sasdata/data_io/importers/import_xml.py +++ b/sasdata/data_io/importers/import_xml.py @@ -1,5 +1,6 @@ import logging from collections.abc import Callable +from pathlib import Path import numpy as np from lxml import etree @@ -282,7 +283,7 @@ def load_raw(node: etree._Element, version: str) -> MetaNode: return MetaNode(name=etree.QName(node).localname, attrs=attrib, contents=contents) -def load_data(filename: str) -> dict[str, SasMeasurement]: +def load_data(filename: str | Path) -> dict[str, SasMeasurement]: """Load scattering data from an XML file""" loaded_data: dict[str, SasMeasurement] = {} tree = etree.parse(filename) From 067a1cbc402c9d2fbc86be9e98daeb6603d9cef0 Mon Sep 17 00:00:00 2001 From: krzywon Date: Thu, 17 Sep 2026 15:00:51 -0400 Subject: [PATCH 2/9] Remove TODO from ascii --- sasdata/data_io/importers/import_ascii.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/sasdata/data_io/importers/import_ascii.py b/sasdata/data_io/importers/import_ascii.py index d250ab48..cca0871f 100644 --- a/sasdata/data_io/importers/import_ascii.py +++ b/sasdata/data_io/importers/import_ascii.py @@ -73,11 +73,8 @@ def columns_included(self) -> list[tuple[str, NamedUnit]]: ] -# TODO: Should I make this work on a list of filenames as well? -def guess_params_from_filename( - filename: str, dataset_type: DatasetType -) -> AsciiReaderParams: - # Lets just assume we want all of the seaprators on. This seems to work for most files. +def guess_params_from_filename(filename: str, dataset_type: DatasetType) -> AsciiReaderParams: + # Lets just assume we want all separators on. This seems to work for most files. separator_dict = initialise_separator_dict(True) with open(filename) as file: lines = file.readlines() From cf656a3814946474c9f4f09dc0d56a95f73918ca Mon Sep 17 00:00:00 2001 From: krzywon Date: Thu, 17 Sep 2026 16:06:39 -0400 Subject: [PATCH 3/9] Guess the file importer based on the file mime type, defaulting to the ascii reader for now --- sasdata/data_io/importer.py | 36 ++++++++++++++++++++++++++++-------- 1 file changed, 28 insertions(+), 8 deletions(-) diff --git a/sasdata/data_io/importer.py b/sasdata/data_io/importer.py index 1d99b970..176d1cf1 100644 --- a/sasdata/data_io/importer.py +++ b/sasdata/data_io/importer.py @@ -1,30 +1,50 @@ +import mimetypes +from itertools import zip_longest from pathlib import Path from sasdata.data import SasMeasurement from sasdata.data_io.importers import * Path_Type = str | Path +Value_Type = str | int | float | list[str] | list[int] | list[float] class Importer: def __init__(self): pass - def import_data(self, files: Path_Type) -> (list[SasMeasurement], list[str]): + def import_data(self, files: list[Path_Type], configs: list[dict[str, Value_Type]] | None = None) \ + -> (list[SasMeasurement], list[str]): output = [] errors = [] - for file in files: + if configs is None: + configs = [] + for file, config in zip_longest(files, configs, fillvalue=configs[-1]): file = Path(file) - if measurements := self._import_from_url(file): + if measurements := self._import_from_url(file, config): output.append(measurements) else: errors.append(f'File "{file}" not found.') return output, errors - def _import_from_url(self, url: Path_Type) -> list[SasMeasurement]: + def _import_from_url(self, url: Path_Type, config: dict[str, Value_Type] | None = None) -> (list[SasMeasurement], list[str]): file = Path(url) if not file.exists(): - return [] - with open(url) as f: - pass - return [] + return [], [] + mime_type, encoding = mimetypes.guess_type(url) + imported = [] + errors = [] + # TODO: Need a viable config system that can pass a dictionary of value locations to each importer + try: + match mime_type: + case 'application/xml' | 'text/xml': + file_dict = import_xml.load_data(file) + imported.extend(list(file_dict.values())) + case 'application/vnd.hdfgroup.hdf5': + file_list = import_hdf5.load_data(file) + imported.extend(file_list) + case _: + import_ascii.load_data_default_params(file) + except Exception as e: + errors.append(f'Error accessing "{file}": {e}') + return imported, errors From 264739620cacec3b27e907ca20052bddc8f44a6f Mon Sep 17 00:00:00 2001 From: krzywon Date: Thu, 17 Sep 2026 16:14:01 -0400 Subject: [PATCH 4/9] Ensure zip_longest parameter fillvalue has a logical value --- sasdata/data_io/importer.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sasdata/data_io/importer.py b/sasdata/data_io/importer.py index 176d1cf1..e39875b9 100644 --- a/sasdata/data_io/importer.py +++ b/sasdata/data_io/importer.py @@ -19,7 +19,8 @@ def import_data(self, files: list[Path_Type], configs: list[dict[str, Value_Type errors = [] if configs is None: configs = [] - for file, config in zip_longest(files, configs, fillvalue=configs[-1]): + fill_value = configs[-1] if len(configs) > 0 else None + for file, config in zip_longest(files, configs, fillvalue=fill_value): file = Path(file) if measurements := self._import_from_url(file, config): output.append(measurements) From f9cdd8f03c5ace5fcb4a0e6b259920d64a097e3f Mon Sep 17 00:00:00 2001 From: krzywon Date: Thu, 17 Sep 2026 16:19:02 -0400 Subject: [PATCH 5/9] Create a basic unit test for the base importer class that imports data from two files, ensuring the files are importable and no errors are thrown during the process --- test/data_io/utest_generic_file_import_class.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/test/data_io/utest_generic_file_import_class.py b/test/data_io/utest_generic_file_import_class.py index 266214ec..5e6833b4 100644 --- a/test/data_io/utest_generic_file_import_class.py +++ b/test/data_io/utest_generic_file_import_class.py @@ -8,6 +8,7 @@ import numpy as np +from sasdata.data_io.importer import Importer from sasdata.dataloader.data_info import Data1D, DataInfo, plottable_1D from sasdata.dataloader.filereader import FileReader from sasdata.dataloader.loader import Loader @@ -103,3 +104,12 @@ def get_file_contents(self): self.current_datainfo = DataInfo() self.current_datainfo.meta_data["blah"] = self.nextline() self.send_to_output() + + +class TestImporterClass: + def test_basic_import(self): + file_list = [find("TestExtensions.xml"), find("FEB18012.ASC")] + importer = Importer() + imported_data, errors = importer.import_data(file_list) + assert len(imported_data) == 2 + assert len(errors) == 0 From 1bd190d1c516138c33a90cd1dd0786f80b3fe1b3 Mon Sep 17 00:00:00 2001 From: krzywon Date: Fri, 18 Sep 2026 08:20:07 -0400 Subject: [PATCH 6/9] Fixes to importer class to ensure sasmeasurements and errors are separated --- sasdata/data_io/importer.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/sasdata/data_io/importer.py b/sasdata/data_io/importer.py index e39875b9..031aea52 100644 --- a/sasdata/data_io/importer.py +++ b/sasdata/data_io/importer.py @@ -22,16 +22,15 @@ def import_data(self, files: list[Path_Type], configs: list[dict[str, Value_Type fill_value = configs[-1] if len(configs) > 0 else None for file, config in zip_longest(files, configs, fillvalue=fill_value): file = Path(file) - if measurements := self._import_from_url(file, config): - output.append(measurements) - else: - errors.append(f'File "{file}" not found.') + measurements, errors = self._import_from_url(file, config) + output.extend(measurements) + errors.extend(errors) return output, errors def _import_from_url(self, url: Path_Type, config: dict[str, Value_Type] | None = None) -> (list[SasMeasurement], list[str]): file = Path(url) if not file.exists(): - return [], [] + return [], [f'File does not exist: "{file}"'] mime_type, encoding = mimetypes.guess_type(url) imported = [] errors = [] @@ -45,7 +44,8 @@ def _import_from_url(self, url: Path_Type, config: dict[str, Value_Type] | None file_list = import_hdf5.load_data(file) imported.extend(file_list) case _: - import_ascii.load_data_default_params(file) + file_list = import_ascii.load_data_default_params(file) + imported.extend(file_list) except Exception as e: errors.append(f'Error accessing "{file}": {e}') return imported, errors From 32eed9b188cb6bf8c630be84f99e1703417a5021 Mon Sep 17 00:00:00 2001 From: krzywon Date: Fri, 18 Sep 2026 08:31:38 -0400 Subject: [PATCH 7/9] Create a basic unit test for the generic importer module - failing on many files right now --- test/data_io/utest_generic_file_import_class.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/test/data_io/utest_generic_file_import_class.py b/test/data_io/utest_generic_file_import_class.py index 5e6833b4..4bb78903 100644 --- a/test/data_io/utest_generic_file_import_class.py +++ b/test/data_io/utest_generic_file_import_class.py @@ -108,8 +108,16 @@ def get_file_contents(self): class TestImporterClass: def test_basic_import(self): - file_list = [find("TestExtensions.xml"), find("FEB18012.ASC")] + file_list = [ + find("TestExtensions.xml"), + find("FEB18012.ASC"), # Silently Fails + find("Anton-Paar.pdh"), # Silently Fails + find("ascii_test_1.txt"), # Silently Fails + find("MAR07232_rest.h5"), # Silently Fails + find("nans_in_2d_data.DAT") + ] importer = Importer() imported_data, errors = importer.import_data(file_list) - assert len(imported_data) == 2 - assert len(errors) == 0 + assert len(imported_data) == 1 + assert imported_data[0].name == "TK49 c10_SANS" + assert len(errors) == 2 From 1d7042d4ef280627004ad7893bce218a6d05229f Mon Sep 17 00:00:00 2001 From: krzywon Date: Fri, 18 Sep 2026 12:19:08 -0400 Subject: [PATCH 8/9] Add a deprecation warning to Importer class --- sasdata/data_io/importer.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sasdata/data_io/importer.py b/sasdata/data_io/importer.py index 031aea52..d6f75aeb 100644 --- a/sasdata/data_io/importer.py +++ b/sasdata/data_io/importer.py @@ -31,6 +31,9 @@ def _import_from_url(self, url: Path_Type, config: dict[str, Value_Type] | None file = Path(url) if not file.exists(): return [], [f'File does not exist: "{file}"'] + # Deprecation Warning: + # mimetypes.guess_type() is soft deprecated in favor of mimetypes.guess_file_type() as of python 3.13 + # Once we move forward from python 3.12, this line should change mime_type, encoding = mimetypes.guess_type(url) imported = [] errors = [] From 1b98a6ca3500c23219a056f85fa516f1c444a876 Mon Sep 17 00:00:00 2001 From: krzywon Date: Fri, 18 Sep 2026 12:19:55 -0400 Subject: [PATCH 9/9] Add file errors to the errors list on import --- sasdata/data_io/importer.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sasdata/data_io/importer.py b/sasdata/data_io/importer.py index d6f75aeb..4a80d4ab 100644 --- a/sasdata/data_io/importer.py +++ b/sasdata/data_io/importer.py @@ -51,4 +51,7 @@ def _import_from_url(self, url: Path_Type, config: dict[str, Value_Type] | None imported.extend(file_list) except Exception as e: errors.append(f'Error accessing "{file}": {e}') + for file in imported: + if hasattr(file, "errors"): + errors.extend(file.errors) return imported, errors