Skip to content

hyfi.project

Project

Bases: BaseConfig, Composer

Project Config

Source code in hyfi/project/project.py 
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
class Project(BaseConfig, Composer):
    """Project Config"""

    _config_name_: str = "__init__"
    _config_group_: str = "/project"
    # Project Config
    project_name: str = "hyfi"
    project_description: Optional[str] = None
    project_root: str = "."
    project_workspace_name: str = "workspace"
    global_hyfi_root: str = "."
    global_workspace_name: str = ".hyfi"
    num_workers: int = 1
    use_huggingface_hub: bool = False
    use_wandb: bool = False
    verbose: Union[bool, int] = False
    # Config Classes
    env: ProjectEnv = None
    joblib: Optional[JobLib] = None
    path: ProjectPath = None

    _property_set_methods_ = {
        "project_name": "set_project_name",
        "project_root": "set_project_root",
    }

    def set_project_root(self, val: Union[str, Path]):
        if not self.project_root or self.project_root != val:
            self.path.project_root = str(val)

    def set_project_name(self, val):
        if not self.project_name or self.project_name != val:
            self.path.project_name = val

    @field_validator("project_name")
    def _validate_project_name(cls, v):
        if v is None:
            raise ValueError("Project name must be specified.")
        return v

    @field_validator("verbose")
    def _validate_verbose(cls, v):
        if isinstance(v, str):
            if v.lower() in {"true", "1"}:
                v = True
            elif v.lower() in {"false", "0"}:
                v = False
            else:
                raise ValueError("verbose must be a boolean or a string of 0 or 1")
        return v

    def __init__(self, **config_kwargs):
        super().__init__(**config_kwargs)
        self.initialize()

    def initialize(self):
        logger.debug("Initializing Project Config: %s", self.project_name)
        self.env = ProjectEnv()

        self.env.HYFI_PROJECT_NAME = self.project_name
        if self.project_description:
            self.env.HYFI_PROJECT_DESC = self.project_description
        if self.project_root:
            self.env.HYFI_PROJECT_ROOT = self.project_root
        if self.project_workspace_name:
            self.env.HYFI_PROJECT_WORKSPACE_NAME = self.project_workspace_name
        if self.global_hyfi_root:
            self.env.HYFI_GLOBAL_ROOT = self.global_hyfi_root
        if self.global_workspace_name:
            self.env.HYFI_GLOBAL_WORKSPACE_NAME = self.global_workspace_name
        if self.num_workers:
            self.env.HYFI_NUM_WORKERS = self.num_workers
        self.env.HYFI_VERBOSE = self.verbose

        if self.joblib:
            self.joblib.init_backend()
        else:
            logger.warning("JoblibConfig not initialized")

        self.init_wandb()
        if self.use_huggingface_hub:
            self.init_huggingface_hub()

    def init_wandb(self):
        if self.path is None:
            raise ValueError("Path object not initialized")
        if self.env is None:
            raise ValueError("Env object not initialized")

        if not self.use_wandb:
            return
        try:
            self._init_wandb()
        except ImportError:
            logger.warning("wandb is not installed, please install it to use wandb.")

    def _init_wandb(self):
        import wandb  # type: ignore

        self.env.WANDB_DIR = str(self.path.log_dir)
        project_name = self.project_name.replace("/", "-").replace("\\", "-")
        self.env.WANDB_PROJECT = project_name
        notebook_name = self.path.log_dir / f"{project_name}-nb"
        notebook_name.mkdir(parents=True, exist_ok=True)
        self.env.WANDB_NOTEBOOK_NAME = str(notebook_name)
        self.env.WANDB_SILENT = str(not self.verbose)

        wandb.init(project=project_name)

    def init_huggingface_hub(self):
        """Initialize huggingface_hub"""
        try:
            from huggingface_hub import notebook_login  # type: ignore
            from huggingface_hub.hf_api import HfFolder  # type: ignore
        except ImportError:
            logger.warning(
                "huggingface_hub is not installed, please install it to use huggingface_hub."
            )
            return

        self.env = ProjectEnv()
        if (
            self.env.HUGGING_FACE_HUB_TOKEN is None
            and self.env.HF_USER_ACCESS_TOKEN is not None
        ):
            self.env.HUGGING_FACE_HUB_TOKEN = self.env.HF_USER_ACCESS_TOKEN

        local_token = HfFolder.get_token()
        if local_token is None:
            if Project.is_notebook():
                notebook_login()
            else:
                logger.info(
                    "huggingface_hub.notebook_login() is only available in notebook,"
                    "set HUGGING_FACE_HUB_TOKEN manually"
                )

    @property
    def osenv(self):
        return os.environ

    @property
    def root_dir(self) -> Path:
        self.path.project_root = self.project_root
        return self.path.root_dir

    @property
    def workspace_dir(self) -> Path:
        self.path.project_workspace_name = self.project_workspace_name
        return self.path.workspace_dir

    @property
    def global_root_dir(self) -> Path:
        self.path.global_hyfi_root = self.global_hyfi_root
        return self.path.global_root_dir

    @property
    def global_workspace_dir(self) -> Path:
        self.path.global_workspace_name = self.global_workspace_name
        return self.path.global_workspace_dir

    def get_path(
        self,
        path_name: str,
        base_dir: Optional[Union[Path, str]] = None,
        ensure_exists: bool = False,
    ) -> Optional[Path]:
        """
        Get the path to a directory or file.
        """
        return (
            self.path.get_path(
                path_name, base_dir=base_dir, ensure_exists=ensure_exists
            )
            if self.path
            else None
        )

get_path(path_name, base_dir=None, ensure_exists=False)

Get the path to a directory or file.

Source code in hyfi/project/project.py 
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
def get_path(
    self,
    path_name: str,
    base_dir: Optional[Union[Path, str]] = None,
    ensure_exists: bool = False,
) -> Optional[Path]:
    """
    Get the path to a directory or file.
    """
    return (
        self.path.get_path(
            path_name, base_dir=base_dir, ensure_exists=ensure_exists
        )
        if self.path
        else None
    )

init_huggingface_hub()

Initialize huggingface_hub

Source code in hyfi/project/project.py 
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
def init_huggingface_hub(self):
    """Initialize huggingface_hub"""
    try:
        from huggingface_hub import notebook_login  # type: ignore
        from huggingface_hub.hf_api import HfFolder  # type: ignore
    except ImportError:
        logger.warning(
            "huggingface_hub is not installed, please install it to use huggingface_hub."
        )
        return

    self.env = ProjectEnv()
    if (
        self.env.HUGGING_FACE_HUB_TOKEN is None
        and self.env.HF_USER_ACCESS_TOKEN is not None
    ):
        self.env.HUGGING_FACE_HUB_TOKEN = self.env.HF_USER_ACCESS_TOKEN

    local_token = HfFolder.get_token()
    if local_token is None:
        if Project.is_notebook():
            notebook_login()
        else:
            logger.info(
                "huggingface_hub.notebook_login() is only available in notebook,"
                "set HUGGING_FACE_HUB_TOKEN manually"
            )

ProjectPath

Bases: BasePath

Source code in hyfi/path/project.py 
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
class ProjectPath(BasePath):
    _config_name_: str = "__project__"

    # internal paths for hyfi
    home: str
    hyfi: str
    resources: str
    runtime: str
    # global paths
    global_hyfi_root: str = "."
    global_workspace_name: str = ".hyfi"
    # project specific paths
    project_name: str
    project_root: str = "."
    project_workspace_name: str = "workspace"

    @property
    def home_dir(self) -> Path:
        """
        Return the path to the home directory.

        Returns:
            path to the home directory
        """
        return Path(self.home).absolute() if self.home else Path.home()

    @property
    def global_root_dir(self) -> Path:
        """
        Create and return the path to the hyfi directory.

        Returns:
            path to the hyfi directory
        """
        path_ = Path(self.global_hyfi_root)
        if not path_.is_absolute():
            path_ = self.home_dir / path_
        return path_

    @property
    def global_workspace_dir(self) -> Path:
        """
        Create and return the path to the glboal workspace directory.

        Returns:
            path to the global workspace directory
        """
        return self.global_root_dir / self.global_workspace_name

    @property
    def global_archive_dir(self) -> Path:
        """
        Create and return the path to the global archive directory.

        Returns:
            path to the global archive directory
        """
        return self.get_path("archives", self.global_workspace_dir)

    @property
    def glboal_dataset_dir(self) -> Path:
        """
        Create and return the path to the global datasets directory.

        Returns:
            path to the global datasets directory
        """
        return self.get_path("datasets", self.global_workspace_dir)

    @property
    def global_model_dir(self) -> Path:
        """
        Create and return the path to the global models directory.

        Returns:
            path to the global models directory
        """
        return self.get_path("models", self.global_workspace_dir)

    @property
    def glboal_module_dir(self) -> Path:
        """
        Create and return the path to the global modules directory.

        Returns:
            path to the global modules directory
        """
        return self.get_path("modules", self.global_workspace_dir)

    @property
    def global_library_dir(self) -> Path:
        """
        Create and return the path to the global library directory.

        Returns:
            path to the global library directory
        """
        return self.get_path("library", self.global_workspace_dir)

    @property
    def glboal_log_dir(self) -> Path:
        """
        Create and return the path to the global log directory.

        Returns:
            path to the global log directory
        """
        return self.get_path("logs", self.global_workspace_dir)

    @property
    def global_cache_dir(self) -> Path:
        """
        Create and return the path to the global cache directory.

        Returns:
            path to the global cache directory
        """
        return self.get_path("cache", self.global_workspace_dir)

    @property
    def global_tmp_dir(self) -> Path:
        """
        Create and return the path to the global tmp directory.

        Returns:
            path to the global tmp directory
        """
        return self.get_path("tmp", self.global_workspace_dir)

    @property
    def name(self) -> str:
        """
        Returns the name of the path configuration.
        """
        return self.project_name

    @property
    def root_dir(self) -> Path:
        """
        Create and return the path to the project directory.

        Returns:
            path to the project directory
        """
        path_ = Path(self.project_root)
        return path_.absolute()

    @property
    def workspace_dir(self) -> Path:
        """
        Create and return the path to the project workspace directory.

        Returns:
            path to the project workspace directory
        """
        return self.root_dir / self.project_workspace_name

glboal_dataset_dir: Path property

Create and return the path to the global datasets directory.

Returns:

Type Description
Path

path to the global datasets directory

glboal_log_dir: Path property

Create and return the path to the global log directory.

Returns:

Type Description
Path

path to the global log directory

glboal_module_dir: Path property

Create and return the path to the global modules directory.

Returns:

Type Description
Path

path to the global modules directory

global_archive_dir: Path property

Create and return the path to the global archive directory.

Returns:

Type Description
Path

path to the global archive directory

global_cache_dir: Path property

Create and return the path to the global cache directory.

Returns:

Type Description
Path

path to the global cache directory

global_library_dir: Path property

Create and return the path to the global library directory.

Returns:

Type Description
Path

path to the global library directory

global_model_dir: Path property

Create and return the path to the global models directory.

Returns:

Type Description
Path

path to the global models directory

global_root_dir: Path property

Create and return the path to the hyfi directory.

Returns:

Type Description
Path

path to the hyfi directory

global_tmp_dir: Path property

Create and return the path to the global tmp directory.

Returns:

Type Description
Path

path to the global tmp directory

global_workspace_dir: Path property

Create and return the path to the glboal workspace directory.

Returns:

Type Description
Path

path to the global workspace directory

home_dir: Path property

Return the path to the home directory.

Returns:

Type Description
Path

path to the home directory

name: str property

Returns the name of the path configuration.

root_dir: Path property

Create and return the path to the project directory.

Returns:

Type Description
Path

path to the project directory

workspace_dir: Path property

Create and return the path to the project workspace directory.

Returns:

Type Description
Path

path to the project workspace directory