diff --git a/collectoss/api/gunicorn_conf.py b/collectoss/api/gunicorn_conf.py index ee7797471..f4eb6216d 100644 --- a/collectoss/api/gunicorn_conf.py +++ b/collectoss/api/gunicorn_conf.py @@ -1,23 +1,16 @@ -# from collectoss import ROOT_PROJECT_REPO_DIRECTORY import multiprocessing import logging -import os from pathlib import Path from glob import glob from collectoss.application.db.lib import get_value from collectoss.application.db import dispose_database_engine from collectoss.application.environment import SystemEnv +from collectoss.application.paths import SystemPaths logger = logging.getLogger(__name__) -# ROOT_PROJECT_REPO_DIRECTORY = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) - -# base_log_dir = ROOT_PROJECT_REPO_DIRECTORY + "/logs/" - -# Path(base_log_dir).mkdir(exist_ok=True) - workers = multiprocessing.cpu_count() * 2 + 1 umask = 0o007 reload = True @@ -39,7 +32,7 @@ del is_dev # set the log location for gunicorn -logs_directory = get_value('Logging', 'logs_directory') +logs_directory = SystemPaths.get_logs_directory() # this syntax satisfies the type checker is_docker = SystemEnv.get_bool("AUGUR_DOCKER_DEPLOY", False) diff --git a/collectoss/application/cli/backend.py b/collectoss/application/cli/backend.py index 3526a3c2c..a621a0675 100644 --- a/collectoss/application/cli/backend.py +++ b/collectoss/application/cli/backend.py @@ -13,6 +13,7 @@ import uuid import traceback import requests +from collectoss.application.paths import SystemPaths from redis.exceptions import ConnectionError as RedisConnectionError from collectoss.application.environment import SystemEnv @@ -101,7 +102,7 @@ def start(ctx, disable_collection, development, pidfile, port): cleanup_collection_status_and_rabbit(logger, ctx.obj.engine) # Retrieve the log directory from the configuration or default to current directory - log_dir = get_value("Logging", "logs_directory") or "." + log_dir = SystemPaths.get_logs_directory() gunicorn_log_file = os.path.join(log_dir, "gunicorn.log") gunicorn_command = f"gunicorn -c {gunicorn_location} -b {host}:{port} collectoss.api.server:app --log-file {gunicorn_log_file}" diff --git a/collectoss/application/logs.py b/collectoss/application/logs.py index aaf6cb5d8..0c9aa2ef9 100644 --- a/collectoss/application/logs.py +++ b/collectoss/application/logs.py @@ -7,6 +7,7 @@ import os from pathlib import Path import shutil +from collectoss.application.paths import SystemPaths import coloredlogs from sqlalchemy.orm import Session @@ -14,9 +15,6 @@ from collectoss.application.config import convert_type_of_value from collectoss.application.db.util import execute_session_query -ROOT_PROJECT_REPO_DIRECTORY = os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__)))) - - SIMPLE_FORMAT_STRING = "[%(process)d] %(name)s [%(levelname)s] %(message)s" VERBOSE_FORMAT_STRING = "%(asctime)s,%(msecs)dms [PID: %(process)d] %(name)s [%(levelname)s] %(message)s" CLI_FORMAT_STRING = "CLI: [%(module)s.%(funcName)s] [%(levelname)s] %(message)s" @@ -117,12 +115,12 @@ def get_log_config(): #TODO dynamically define loggers for every task names. class TaskLogConfig(): - def __init__(self, all_tasks, disable_log_files=False,reset_logfiles=False,base_log_dir=ROOT_PROJECT_REPO_DIRECTORY + "/logs/"): + def __init__(self, all_tasks, disable_log_files=False, reset_logfiles=False, base_log_dir=None): log_config = get_log_config() - if log_config["logs_directory"] != "": - base_log_dir=log_config["logs_directory"] + if not base_log_dir: + base_log_dir = SystemPaths.get_logs_directory() if reset_logfiles is True: try: @@ -188,12 +186,12 @@ def getLoggerNames(self): class SystemLogger(): - def __init__(self, logger_name, disable_log_files=False,reset_logfiles=False,base_log_dir=ROOT_PROJECT_REPO_DIRECTORY + "/logs/"): + def __init__(self, logger_name, disable_log_files=False, reset_logfiles=False, base_log_dir=None): log_config = get_log_config() - if log_config.get("logs_directory", "") != "": - base_log_dir=log_config.get("logs_directory") + if not base_log_dir: + base_log_dir = SystemPaths.get_logs_directory() if reset_logfiles is True: try: diff --git a/collectoss/application/paths.py b/collectoss/application/paths.py new file mode 100644 index 000000000..3d810ff9a --- /dev/null +++ b/collectoss/application/paths.py @@ -0,0 +1,181 @@ +from platformdirs import PlatformDirs +from collectoss.application.environment import SystemEnv +from pathlib import Path + +import logging + +logger = logging.getLogger(__name__) + +def _clean_path(path: Path | str) -> Path | None: + if path is None: + return None + if isinstance(path, str): + path = Path(path) + return path.expanduser().resolve() + +def _verify_path(path: Path, create = True) -> Path | None: + """Verify the path is a valid directory""" + if create: + if not path.exists(): + path.mkdir(parents=True) + if not path.is_dir(): + raise ValueError(f"Path {path} is not a valid directory") + return _clean_path(path) + + +def _path_from_env(env_value: str) -> Path | None: + """Get the path from the environment variable""" + if env_value is None: + return None + if env_value == "": + return None + return Path(env_value) + +def _build_path(env_path:str, default_path:Path) -> Path: + """Build a path from the environment variable or the default path. + + If the environment variable is an absolute path, return it. + If the environment variable is a relative path, resolve it against the home directory. + If the environment variable is not set, return the default path. + """ + if env_path is not None: + env_path = Path(env_path) + if env_path.is_absolute(): + return _clean_path(env_path) + else: + return _clean_path(Path.home() / env_path) + else: + return default_path + +class SystemPaths: + """Enable consistent storage and retrieval of filesystem paths needed by the system + + The paths that are used follow the following hierarchy: + - Absolute path specified by an environment variable + - Relative path specified by an environment variable, resolved against the home directory + - Default path for the operating system based on accepted standards + + """ + app_name = "CollectOSS" + app_org = "CHAOSS" + + @staticmethod + def os_defaults(create = True) -> PlatformDirs: + """Get the set of conventional directories for the operating system""" + return PlatformDirs(SystemPaths.app_name, SystemPaths.app_org, ensure_exists=create) + + @staticmethod + def get_facade_directory(create = True) -> Path: + """Get the facade directory. Requires database for historical compatibility""" + env_path = _path_from_env(SystemEnv.get("COLLECTOSS_FACADE_REPO_DIRECTORY")) + database_path = None + + from collectoss.application.config import SystemConfig + from collectoss.application.db.session import DatabaseSession + from collectoss.application.db import get_engine + with DatabaseSession(logger, get_engine()) as session: + config = SystemConfig(logger, session) + database_path = config.get_value("Facade", "repo_directory") + + + return _verify_path( + _build_path(env_path or database_path, SystemPaths.os_defaults(create).user_downloads_path / "collectoss_facade"), + create = create + ) + + @staticmethod + def facade_repo_path(repo) -> Path: + """Get the path to a specific facade repository""" + return SystemPaths.get_facade_directory() / f"{repo.repo_id}-{repo.repo_path}/{repo.repo_name}" + + @staticmethod + def get_config_directory(create = True) -> Path: + """Get the config directory""" + env_path = _path_from_env(SystemEnv.get("COLLECTOSS_CONFIG_DIRECTORY") or SystemEnv.get("CONFIG_DATADIR")) + + return _verify_path( + _build_path(env_path, SystemPaths.os_defaults(create).user_config_path), + create = create + ) + + @staticmethod + def get_logs_directory(create = True) -> Path: + """Get the logs directory. Requires database for historical compatibility""" + env_path = _path_from_env(SystemEnv.get("COLLECTOSS_LOGS_DIRECTORY")) + database_path = None + + from collectoss.application.config import SystemConfig + from collectoss.application.db.session import DatabaseSession + from collectoss.application.db import get_engine + with DatabaseSession(logger, get_engine()) as session: + config = SystemConfig(logger, session) + database_path = config.get_value("Logging", "logs_directory") + + return _verify_path( + _build_path(env_path or database_path, SystemPaths.os_defaults(create).user_log_path), + create = create + ) + + @staticmethod + def get_cache_directory(create = True) -> Path: + """Get the cache directory""" + env_path = _path_from_env(SystemEnv.get("COLLECTOSS_CACHE_DIRECTORY") or SystemEnv.get("CACHE_DATADIR")) + + return _verify_path( + _build_path(env_path, SystemPaths.os_defaults(create).user_cache_path), + create = create + ) + + + @staticmethod + def get_models_directory(create = True) -> Path: + """Get the models directory. Requires database for historical compatibility""" + database_dirname = None + + from collectoss.application.config import SystemConfig + from collectoss.application.db.session import DatabaseSession + from collectoss.application.db import get_engine + with DatabaseSession(logger, get_engine()) as session: + config = SystemConfig(logger, session) + database_dirname = config.get_value("Message_Insights", 'models_dir') or "message_models" + + return _verify_path( + SystemPaths.os_defaults(create).user_data_path / "tasks" / "data_analysis" / "message_insights" / database_dirname, + create = create + ) + + @staticmethod + def get_model_training_data_directory(create = True) -> Path: + """Get the model training data directory""" + env_path = _path_from_env(SystemEnv.get("COLLECTOSS_ANALYSIS_DIRECTORY")) + return _verify_path( + _build_path(env_path / "message_insights" / "train_data", SystemPaths.os_defaults(create).user_data_path / "tasks" / "data_analysis" / "message_insights" / "train_data"), + create = create + ) + + @staticmethod + def get_discourse_analysis_directory(create = True) -> Path: + """Get the discourse analysis directory""" + env_path = _path_from_env(SystemEnv.get("COLLECTOSS_ANALYSIS_DIRECTORY")) + return _verify_path( + _build_path(env_path / "discourse_analysis", SystemPaths.os_defaults(create).user_data_path / "tasks" / "data_analysis" / "discourse_analysis"), + create = create + ) + + @staticmethod + def get_install_path() -> Path: + """Get the path that CollectOSS is currently installed to. This should be treated as read- only.""" + # This paths file is only one level below the root of the module. + # accessing above that is not possible as the module could be installed separately + return _verify_path(Path(__file__).parent, create = False) + + @staticmethod + def print_all_paths(logger): + logger.info(f"Install path: {SystemPaths.get_install_path()}") + logger.info(f"Facade directory: {SystemPaths.get_facade_directory(create = False)}") + logger.info(f"Config directory: {SystemPaths.get_config_directory(create = False)}") + logger.info(f"Logs directory: {SystemPaths.get_logs_directory(create = False)}") + logger.info(f"Cache directory: {SystemPaths.get_cache_directory(create = False)}") + logger.info(f"Models directory: {SystemPaths.get_models_directory(create = False)}") + logger.info(f"Model training data directory: {SystemPaths.get_model_training_data_directory(create = False)}") + logger.info(f"Discourse analysis directory: {SystemPaths.get_discourse_analysis_directory(create = False)}") \ No newline at end of file diff --git a/collectoss/tasks/data_analysis/discourse_analysis/tasks.py b/collectoss/tasks/data_analysis/discourse_analysis/tasks.py index fccf169e8..b026f2774 100644 --- a/collectoss/tasks/data_analysis/discourse_analysis/tasks.py +++ b/collectoss/tasks/data_analysis/discourse_analysis/tasks.py @@ -1,4 +1,5 @@ import logging +from collectoss.application.paths import SystemPaths import sqlalchemy as s import pandas as pd import pickle @@ -29,8 +30,7 @@ # from os import path stemmer = nltk.stem.snowball.SnowballStemmer("english") -ROOT_PROJECT_REPO_DIRECTORY = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__))))) -DISCOURSE_ANALYSIS_DIR = f"{ROOT_PROJECT_REPO_DIRECTORY}/tasks/data_analysis/discourse_analysis/" +DISCOURSE_ANALYSIS_DIR = SystemPaths.get_discourse_analysis_directory() @celery.task(base=MLRepoCollectionTask, bind=True) def discourse_analysis_task(self, repo_git): diff --git a/collectoss/tasks/data_analysis/message_insights/message_novelty.py b/collectoss/tasks/data_analysis/message_insights/message_novelty.py index bb5531b04..dfed5a3fd 100644 --- a/collectoss/tasks/data_analysis/message_insights/message_novelty.py +++ b/collectoss/tasks/data_analysis/message_insights/message_novelty.py @@ -5,6 +5,7 @@ import os from datetime import datetime, timedelta +from collectoss.application.paths import SystemPaths import numpy as np import pandas as pd from gensim.models.doc2vec import Doc2Vec, TaggedDocument @@ -16,10 +17,7 @@ from collectoss.tasks.data_analysis.message_insights.preprocess_text import \ normalize_corpus as normalize_corpus -ROOT_PROJECT_REPO_DIRECTORY = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__))))) - - -train_path = os.path.join(ROOT_PROJECT_REPO_DIRECTORY, "tasks", "data_analysis", "message_insights", "train_data") +train_path = SystemPaths.get_model_training_data_directory() # ''' Doc2Vec model training diff --git a/collectoss/tasks/data_analysis/message_insights/message_sentiment.py b/collectoss/tasks/data_analysis/message_insights/message_sentiment.py index 4ce60c7d6..6bc25680e 100644 --- a/collectoss/tasks/data_analysis/message_insights/message_sentiment.py +++ b/collectoss/tasks/data_analysis/message_insights/message_sentiment.py @@ -10,6 +10,7 @@ import warnings from statistics import mean +from collectoss.application.paths import SystemPaths import emoji import joblib import nltk @@ -30,11 +31,9 @@ warnings.filterwarnings('ignore') -ROOT_PROJECT_REPO_DIRECTORY = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__))))) - CONTRACTION_MAP = contraction_map -train_path = os.path.join(ROOT_PROJECT_REPO_DIRECTORY, "tasks", "data_analysis", "message_insights", "train_data") +train_path = SystemPaths.get_model_training_data_directory() def replace_all(text, dic): if(sys.version_info[0] < 3): diff --git a/collectoss/tasks/data_analysis/message_insights/tasks.py b/collectoss/tasks/data_analysis/message_insights/tasks.py index 751da1ea7..1e3abb21f 100644 --- a/collectoss/tasks/data_analysis/message_insights/tasks.py +++ b/collectoss/tasks/data_analysis/message_insights/tasks.py @@ -1,6 +1,7 @@ import datetime import logging import os +from collectoss.application.paths import SystemPaths import numpy as np import pandas as pd import requests @@ -18,8 +19,6 @@ #SPDX-License-Identifier: MIT -ROOT_PROJECT_REPO_DIRECTORY = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__))))) - @celery.task(base=MLRepoCollectionTask, bind=True) def message_insight_task(self, repo_git): @@ -45,7 +44,7 @@ def message_insight_model(repo_git: str,logger,engine) -> None: repo = get_repo_by_repo_git(repo_git) repo_id = repo.repo_id - models_dir = os.path.join(ROOT_PROJECT_REPO_DIRECTORY, "tasks", "data_analysis", "message_insights", get_value("Message_Insights", 'models_dir')) + models_dir = SystemPaths.get_models_directory() insight_days = get_value("Message_Insights", 'insight_days') # Any initial database instructions, like finding the last tuple inserted or generate the next ID value diff --git a/collectoss/tasks/data_analysis/pull_request_analysis_worker/tasks.py b/collectoss/tasks/data_analysis/pull_request_analysis_worker/tasks.py index 34512fbc9..bd4c3d1fa 100644 --- a/collectoss/tasks/data_analysis/pull_request_analysis_worker/tasks.py +++ b/collectoss/tasks/data_analysis/pull_request_analysis_worker/tasks.py @@ -1,7 +1,7 @@ import logging -import os import datetime +from collectoss.application.paths import SystemPaths import joblib import pandas as pd import sqlalchemy as s @@ -18,8 +18,6 @@ # from sklearn.preprocessing import LabelEncoder, MinMaxScaler # from xgboost import XGBClassifier -ROOT_PROJECT_REPO_DIRECTORY = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__))))) - @celery.task(base=MLRepoCollectionTask, bind=True) def pull_request_analysis_task(self, repo_git): @@ -40,7 +38,7 @@ def pull_request_analysis_model(repo_git: str,logger,engine) -> None: repo_id = get_repo_by_repo_git(repo_git).repo_id - senti_models_dir = os.path.join(ROOT_PROJECT_REPO_DIRECTORY, "tasks", "data_analysis", "message_insights", get_value("Message_Insights", 'models_dir')) + senti_models_dir = SystemPaths.get_models_directory() logger.info(f'Sentiment model dir located - {senti_models_dir}') diff --git a/collectoss/tasks/git/dependency_libyear_tasks/core.py b/collectoss/tasks/git/dependency_libyear_tasks/core.py index 56b8f1a5b..fb6ec3f7b 100644 --- a/collectoss/tasks/git/dependency_libyear_tasks/core.py +++ b/collectoss/tasks/git/dependency_libyear_tasks/core.py @@ -1,6 +1,7 @@ from datetime import datetime from collectoss.application.db.models import * -from collectoss.application.db.lib import get_value, bulk_insert_dicts, get_repo_by_repo_git +from collectoss.application.db.lib import bulk_insert_dicts, get_repo_by_repo_git +from collectoss.application.paths import SystemPaths from collectoss.tasks.git.dependency_libyear_tasks.libyear_util.util import get_deps_libyear_data from collectoss.tasks.git.util.facade_worker.facade_worker.utilitymethods import get_absolute_repo_path @@ -15,7 +16,7 @@ def deps_libyear_model(logger,repo_git): repo = get_repo_by_repo_git(repo_git) - absolute_repo_path = get_absolute_repo_path(get_value("Facade", "repo_directory"),repo.repo_id,repo.repo_path,repo.repo_name) + absolute_repo_path = get_absolute_repo_path(SystemPaths.get_facade_directory(),repo.repo_id,repo.repo_path,repo.repo_name) #config.get_section("Facade")['repo_directory'] + relative_repo_path#self.config['repo_directory'] + relative_repo_path generate_deps_libyear_data(logger, repo.repo_id, absolute_repo_path) diff --git a/collectoss/tasks/git/dependency_tasks/core.py b/collectoss/tasks/git/dependency_tasks/core.py index 0648231b0..78c797b22 100644 --- a/collectoss/tasks/git/dependency_tasks/core.py +++ b/collectoss/tasks/git/dependency_tasks/core.py @@ -1,8 +1,9 @@ from datetime import datetime import os from collectoss.application.db.models import * -from collectoss.application.db.lib import bulk_insert_dicts, get_repo_by_repo_git, get_value +from collectoss.application.db.lib import bulk_insert_dicts, get_repo_by_repo_git from collectoss.application.environment import SystemEnv +from collectoss.application.paths import SystemPaths from collectoss.tasks.github.util.github_api_key_handler import GithubApiKeyHandler from collectoss.tasks.git.dependency_tasks.dependency_util import dependency_calculator as dep_calc from collectoss.tasks.util.worker_util import parse_json_from_subprocess_call @@ -22,7 +23,7 @@ def generate_deps_data(logger, repo_git): repo = get_repo_by_repo_git(repo_git) repo_id = repo.repo_id - path = get_absolute_repo_path(get_value("Facade", "repo_directory"),repo.repo_id,repo.repo_path,repo.repo_name) + path = get_absolute_repo_path(SystemPaths.get_facade_directory(),repo.repo_id,repo.repo_path,repo.repo_name) logger.debug(f"This is the deps model repo: {repo_git}.") diff --git a/collectoss/tasks/git/scc_value_tasks/core.py b/collectoss/tasks/git/scc_value_tasks/core.py index 770165522..63d9f079c 100644 --- a/collectoss/tasks/git/scc_value_tasks/core.py +++ b/collectoss/tasks/git/scc_value_tasks/core.py @@ -1,8 +1,9 @@ from datetime import datetime import os from collectoss.application.db.models import * -from collectoss.application.db.lib import bulk_insert_dicts, get_repo_by_repo_git, get_value +from collectoss.application.db.lib import bulk_insert_dicts, get_repo_by_repo_git from collectoss.application.environment import SystemEnv +from collectoss.application.paths import SystemPaths from collectoss.tasks.util.worker_util import parse_json_from_subprocess_call from collectoss.tasks.git.util.facade_worker.facade_worker.utilitymethods import get_absolute_repo_path @@ -15,7 +16,7 @@ def value_model(logger,repo_git): repo = get_repo_by_repo_git(repo_git) repo_id = repo.repo_id - path = get_absolute_repo_path(get_value("Facade", "repo_directory"),repo_id,repo.repo_path,repo.repo_name) + path = get_absolute_repo_path(SystemPaths.get_facade_directory(),repo_id,repo.repo_path,repo.repo_name) logger.info('Generating value data for repo') logger.info(f"Repo ID: {repo_id}, Path: {path}") diff --git a/collectoss/tasks/git/util/facade_worker/facade_worker/config.py b/collectoss/tasks/git/util/facade_worker/facade_worker/config.py index 2b536a3a4..b546df243 100644 --- a/collectoss/tasks/git/util/facade_worker/facade_worker/config.py +++ b/collectoss/tasks/git/util/facade_worker/facade_worker/config.py @@ -31,6 +31,7 @@ import random import subprocess from urllib.parse import urlparse +from collectoss.application.paths import SystemPaths import sqlalchemy as s from sqlalchemy.exc import OperationalError from psycopg2.errors import DeadlockDetected @@ -140,15 +141,12 @@ def __init__(self,logger: Logger): self.tool_version = "1.4.4" # Get the location of the directory where git repos are stored - if 'repo_directory' in worker_options: - self.repo_base_directory = worker_options['repo_directory'] - else: - self.repo_base_directory = None + self.repo_base_directory = SystemPaths.get_facade_directory() # Determine if it's safe to start the script current_status = self.get_setting('utility_status') - if len(self.repo_base_directory) == 0: + if len(str(self.repo_base_directory)) == 0: self.cfg.log_activity('Error','No base directory. It is unsafe to continue.') raise Exception('Failed: No base directory') diff --git a/collectoss/tasks/git/util/facade_worker/facade_worker/utilitymethods.py b/collectoss/tasks/git/util/facade_worker/facade_worker/utilitymethods.py index afba70fa2..5d5e3dfcc 100644 --- a/collectoss/tasks/git/util/facade_worker/facade_worker/utilitymethods.py +++ b/collectoss/tasks/git/util/facade_worker/facade_worker/utilitymethods.py @@ -102,6 +102,7 @@ def trim_author(facade_helper, email): facade_helper.log_activity('Debug',f"Trimmed working author: {email}") +@deprecated("This method of getting the absolute repo path is deprecated. Use SystemPaths.facade_repo_path(repo) instead.") def get_absolute_repo_path(repo_base_dir, repo_id, repo_path,repo_name): return f"{repo_base_dir}{repo_id}-{repo_path}/{repo_name}" diff --git a/collectoss/util/startup.py b/collectoss/util/startup.py index 8fe5b2374..4b61888af 100644 --- a/collectoss/util/startup.py +++ b/collectoss/util/startup.py @@ -9,6 +9,7 @@ import platform import sys +from collectoss.application.paths import SystemPaths from sqlalchemy.orm.attributes import get_history from collectoss.application.config import SystemConfig from collectoss.application.db.session import DatabaseSession @@ -17,8 +18,6 @@ from collectoss.util.inspect_without_import import get_phase_names_without_import -ROOT_PROJECT_REPO_DIRECTORY = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__))))) - def check_init_schema(): """Initialize the CollectOSS database schema as appropriate """ @@ -103,14 +102,7 @@ def collect_env_variables(logger): def setup_facade_directory(logger): """Perform permission checks and create the facade directory if it doesnt exist """ - - facade_directory_path = SystemEnv.get("COLLECTOSS_FACADE_REPO_DIRECTORY") or "/collectoss/facade/" - - facade_directory = Path(facade_directory_path) - - if not facade_directory.exists(): - logger.debug(f"Specified facade directory {facade_directory_path} does not exist. Creating...") - facade_directory.mkdir() + facade_directory = SystemPaths.get_facade_directory() git_credentials = facade_directory.joinpath(".git-credentials") git_credentials.touch(exist_ok=True) @@ -213,9 +205,11 @@ def merge_config( augmented_config["Keys"] = keys - augmented_config["Facade"]["repo_directory"] = facade_repo_directory - - augmented_config["Logging"]["logs_directory"] = logs_directory or (ROOT_PROJECT_REPO_DIRECTORY + "/logs/") + if facade_repo_directory and facade_repo_directory != "": + augmented_config["Facade"]["repo_directory"] = facade_repo_directory + + if logs_directory and logs_directory != "": + augmented_config["Logging"]["logs_directory"] = logs_directory config.load_config_from_dict(augmented_config) diff --git a/docker/backend/preflight.py b/docker/backend/preflight.py index 4207db8b7..10f7130aa 100755 --- a/docker/backend/preflight.py +++ b/docker/backend/preflight.py @@ -1,3 +1,4 @@ +from collectoss.application.paths import SystemPaths from collectoss.util.startup import collect_env_variables, check_init_schema, check_update_schema, setup_facade_directory, merge_config, warn_import_repos, print_platform_information from collectoss.application.logs import getFormatter from collectoss.application.cli import DatabaseContext @@ -31,5 +32,6 @@ warn_import_repos(logger) print_platform_information(logger) + SystemPaths.print_all_paths(logger) sys.exit(0) diff --git a/tests/test_classes/test_paths.py b/tests/test_classes/test_paths.py new file mode 100644 index 000000000..d1ac250dd --- /dev/null +++ b/tests/test_classes/test_paths.py @@ -0,0 +1,8 @@ +from collectoss.application.paths import _build_path +from pathlib import Path + +class TestBuildPath: + def test_build_path(self): + assert _build_path(None, Path("/path")) == Path("/path") + assert _build_path("collectoss", Path("/path")) == Path.home() / "collectoss" + assert _build_path("/collectoss", Path("/path")) == Path("/collectoss") \ No newline at end of file