From cbec53af34cc0d16f0bf67e8c6278ec4f7ac02fb Mon Sep 17 00:00:00 2001 From: "Yuyao Huang (Sam)" Date: Thu, 29 May 2025 18:32:58 +0800 Subject: [PATCH] =?UTF-8?q?=E8=BF=81=E7=A7=BB=20lazy=5Fconfig?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .vscode/settings.json | 18 + LICENSE | 181 ++++++++++ README.md | 28 +- lazy_config/__init__.py | 0 lazy_config/utils/__init__.py | 0 lazy_config/utils/config/__init__.py | 10 + lazy_config/utils/config/argparser.py | 28 ++ lazy_config/utils/config/instantiate.py | 101 ++++++ lazy_config/utils/config/lazy.py | 449 ++++++++++++++++++++++++ lazy_config/utils/file_io.py | 54 +++ lazy_config/utils/registry.py | 73 ++++ projects/.gitkeep | 0 projects/lazy_config_demo/config.py | 9 + projects/lazy_config_demo/main.py | 11 + projects/lazy_config_demo/model.py | 14 + pyproject.toml | 38 ++ 16 files changed, 1013 insertions(+), 1 deletion(-) create mode 100644 .vscode/settings.json create mode 100644 LICENSE create mode 100644 lazy_config/__init__.py create mode 100644 lazy_config/utils/__init__.py create mode 100644 lazy_config/utils/config/__init__.py create mode 100644 lazy_config/utils/config/argparser.py create mode 100644 lazy_config/utils/config/instantiate.py create mode 100644 lazy_config/utils/config/lazy.py create mode 100644 lazy_config/utils/file_io.py create mode 100644 lazy_config/utils/registry.py create mode 100644 projects/.gitkeep create mode 100644 projects/lazy_config_demo/config.py create mode 100644 projects/lazy_config_demo/main.py create mode 100644 projects/lazy_config_demo/model.py create mode 100644 pyproject.toml diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..fa9f3cc --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,18 @@ +{ + "workbench.colorCustomizations": { + "titleBar.activeBackground": "#ffb5d5", + "titleBar.activeForeground": "#000000", + "titleBar.inactiveBackground": "#ffb5d5", + "titleBar.inactiveForeground": "#000000", + "titleBar.border": "#ffb5d5", + "activityBar.background": "#ffb5d5", + "activityBar.foreground": "#000000", + "statusBar.background": "#ffb5d5", + "statusBar.foreground": "#000000", + "statusBar.debuggingBackground": "#ffb5d5", + "statusBar.debuggingForeground": "#000000", + "tab.activeBorder": "#ffb5d5", + "iLoveWorkSpaceColors": true, + "iLoveWorkSpaceRandom": false + } +} \ No newline at end of file diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..d3a8ad4 --- /dev/null +++ b/LICENSE @@ -0,0 +1,181 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the Work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + copyrighted components of the original Work have been recombined, + rearranged, modified, or otherwise modified. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or Derivative Works + a copy of this License; and + + (b) You must cause any modified files to carry prominent notices stating + that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works that You + distribute, all copyright, patent, trademark, and attribution notices + from the Source form of the Work, excluding those notices that do not + pertain to any part of the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + where such third-party notices normally appear. The contents of the + NOTICE file are for informational purposes only and do not modify + the License. You may add Your own attribution notices within + Derivative Works that You distribute, alongside, or as an addendum to + the NOTICE text from the Work, provided that such additional + attribution notices cannot be construed as modifying the License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + +To apply the Apache License to your work, attach the following +boilerplate notice, with the fields enclosed by brackets "[]" replaced +with your own identifying information. (Don't include the brackets!) +The text should be enclosed in the appropriate comment syntax for +the file format. We also recommend that a file or class name and +description of purpose be included on the same "printed page" as the +copyright notice for easier identification within third-party archives. + +Copyright 2025 Yuyao Huang + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/README.md b/README.md index a905972..9a0c047 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,29 @@ # lazy_config -剥离 detectron2 的 LazyConfig 功能，用于通用任务的项目模板。 \ No newline at end of file +剥离 detectron2 的 LazyConfig 功能，用于通用任务的项目模板。 + +## How to create your own package based on this package. + +```bash +python init.py +``` + +and follow the prompts. + +After the project is created, you can delete the `init.py` file and start working on your project. + +## Usage + +See the example in `projects/lazy_config_demo/main.py` and run it with the following command: + +``` +PYTHONPATH=. python projects/lazy_config_demo/main.py projects/lazy_config_demo/config.py MODEL.in_features=10 +``` + +## Principles + +- `LazyCall` (or `L`): 这是让你能够以惰性方式定义对象构建的语法糖。 + +- `instantiate()` function: 这是将惰性配置（带有 _target_ 的 omegaconf 对象）转换为实际 Python 对象的关键函数。它会处理递归实例化和参数传递。 + +- `parse_args_and_configs()` function: 它允许你读取命令行参数和配置文件，并将它们合并成一个 omegaconf 对象。 \ No newline at end of file diff --git a/lazy_config/__init__.py b/lazy_config/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/lazy_config/utils/__init__.py b/lazy_config/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/lazy_config/utils/config/__init__.py b/lazy_config/utils/config/__init__.py new file mode 100644 index 0000000..3c4f17f --- /dev/null +++ b/lazy_config/utils/config/__init__.py @@ -0,0 +1,10 @@ +from .instantiate import instantiate +from .lazy import LazyCall, LazyConfig +from .argparser import parse_args_and_configs + +__all__ = [ + "instantiate", + "LazyCall", + "LazyConfig", + "parse_args_and_configs", +] \ No newline at end of file diff --git a/lazy_config/utils/config/argparser.py b/lazy_config/utils/config/argparser.py new file mode 100644 index 0000000..6e436cb --- /dev/null +++ b/lazy_config/utils/config/argparser.py @@ -0,0 +1,28 @@ +import argparse +import os +from .lazy import LazyConfig + + +def parse_args_and_configs(program_description: str = ""): + parser = argparse.ArgumentParser(description=program_description) + parser.add_argument( + "config_file", + metavar="FILE", + help="path to config file", + ) + parser.add_argument( + "opts", + help="Modify config options using the command-line 'KEY VALUE' pairs", + default=None, + nargs=argparse.REMAINDER, + ) + args = parser.parse_args() + + if not os.path.exists(args.config_file): + raise FileNotFoundError(f"Config file not found: {args.config_file}") + cfg = LazyConfig.load(args.config_file) + + if args.opts: + cfg.merge_with_dotlist(args.opts) + + return cfg \ No newline at end of file diff --git a/lazy_config/utils/config/instantiate.py b/lazy_config/utils/config/instantiate.py new file mode 100644 index 0000000..49fca42 --- /dev/null +++ b/lazy_config/utils/config/instantiate.py @@ -0,0 +1,101 @@ +# -*- coding: utf-8 -*- +# Original Copyright (c) Facebook, Inc. and its affiliates. +# This file has been modified by Yuyao Huang (C) 2025. +# SPDX-License-Identifier: Apache-2.0 +# +# This file is based on code from Detectron2 (https://github.com/facebookresearch/detectron2) +# and is licensed under the Apache License, Version 2.0. +# You may obtain a copy of the License at (http://www.apache.org/licenses/LICENSE-2.0). +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import collections.abc as abc +import dataclasses +import logging +from typing import Any + +from lazy_config.utils.registry import _convert_target_to_string, locate + +__all__ = ["dump_dataclass", "instantiate"] + + +def dump_dataclass(obj: Any): + """ + Dump a dataclass recursively into a dict that can be later instantiated. + + Args: + obj: a dataclass object + + Returns: + dict + """ + assert dataclasses.is_dataclass(obj) and not isinstance( + obj, type + ), "dump_dataclass() requires an instance of a dataclass." + ret = {"_target_": _convert_target_to_string(type(obj))} + for f in dataclasses.fields(obj): + v = getattr(obj, f.name) + if dataclasses.is_dataclass(v): + v = dump_dataclass(v) + if isinstance(v, (list, tuple)): + v = [dump_dataclass(x) if dataclasses.is_dataclass(x) else x for x in v] + ret[f.name] = v + return ret + + +def instantiate(cfg): + """ + Recursively instantiate objects defined in dictionaries by + "_target_" and arguments. + + Args: + cfg: a dict-like object with "_target_" that defines the caller, and + other keys that define the arguments + + Returns: + object instantiated by cfg + """ + from omegaconf import ListConfig, DictConfig, OmegaConf + + if isinstance(cfg, ListConfig): + lst = [instantiate(x) for x in cfg] + return ListConfig(lst, flags={"allow_objects": True}) + if isinstance(cfg, list): + # Specialize for list, because many classes take + # list[objects] as arguments, such as ResNet, DatasetMapper + return [instantiate(x) for x in cfg] + + # If input is a DictConfig backed by dataclasses (i.e. omegaconf's structured config), + # instantiate it to the actual dataclass. + if isinstance(cfg, DictConfig) and dataclasses.is_dataclass(cfg._metadata.object_type): + return OmegaConf.to_object(cfg) + + if isinstance(cfg, abc.Mapping) and "_target_" in cfg: + # conceptually equivalent to hydra.utils.instantiate(cfg) with _convert_=all, + # but faster: https://github.com/facebookresearch/hydra/issues/1200 + cfg = {k: instantiate(v) for k, v in cfg.items()} + cls = cfg.pop("_target_") + cls = instantiate(cls) + + if isinstance(cls, str): + cls_name = cls + cls = locate(cls_name) + assert cls is not None, cls_name + else: + try: + cls_name = cls.__module__ + "." + cls.__qualname__ + except Exception: + # target could be anything, so the above could fail + cls_name = str(cls) + assert callable(cls), f"_target_ {cls} does not define a callable object" + try: + return cls(**cfg) + except TypeError: + logger = logging.getLogger(__name__) + logger.error(f"Error when instantiating {cls_name}!") + raise + return cfg # return as-is if don't know what to do \ No newline at end of file diff --git a/lazy_config/utils/config/lazy.py b/lazy_config/utils/config/lazy.py new file mode 100644 index 0000000..0edeb70 --- /dev/null +++ b/lazy_config/utils/config/lazy.py @@ -0,0 +1,449 @@ +# -*- coding: utf-8 -*- +# Original Copyright (c) Facebook, Inc. and its affiliates. +# This file has been modified by Yuyao Huang (C) 2025. +# SPDX-License-Identifier: Apache-2.0 +# +# This file is based on code from Detectron2 (https://github.com/facebookresearch/detectron2) +# and is licensed under the Apache License, Version 2.0. +# You may obtain a copy of the License at (http://www.apache.org/licenses/LICENSE-2.0). +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import ast +import builtins +import collections.abc as abc +import importlib +import inspect +import logging +import os +import uuid +from contextlib import contextmanager +from copy import deepcopy +from dataclasses import is_dataclass +from typing import List, Tuple, Union +import cloudpickle +import yaml +from omegaconf import DictConfig, ListConfig, OmegaConf, SCMode + +from lazy_config.utils.file_io import PathManager +from lazy_config.utils.registry import _convert_target_to_string + +__all__ = ["LazyCall", "LazyConfig"] + + +class LazyCall: + """ + Wrap a callable so that when it's called, the call will not be executed, + but returns a dict that describes the call. + + LazyCall object has to be called with only keyword arguments. Positional + arguments are not yet supported. + + Examples: + :: + from detectron2.config import instantiate, LazyCall + + layer_cfg = LazyCall(nn.Conv2d)(in_channels=32, out_channels=32) + layer_cfg.out_channels = 64 # can edit it afterwards + layer = instantiate(layer_cfg) + """ + + def __init__(self, target): + if not (callable(target) or isinstance(target, (str, abc.Mapping))): + raise TypeError( + f"target of LazyCall must be a callable or defines a callable! Got {target}" + ) + self._target = target + + def __call__(self, **kwargs): + if is_dataclass(self._target): + # omegaconf object cannot hold dataclass type + # https://github.com/omry/omegaconf/issues/784 + target = _convert_target_to_string(self._target) + else: + target = self._target + kwargs["_target_"] = target + + return DictConfig(content=kwargs, flags={"allow_objects": True}) + + +def _visit_dict_config(cfg, func): + """ + Apply func recursively to all DictConfig in cfg. + """ + if isinstance(cfg, DictConfig): + func(cfg) + for v in cfg.values(): + _visit_dict_config(v, func) + elif isinstance(cfg, ListConfig): + for v in cfg: + _visit_dict_config(v, func) + + +def _validate_py_syntax(filename): + # see also https://github.com/open-mmlab/mmcv/blob/master/mmcv/utils/config.py + with PathManager.open(filename, "r") as f: + content = f.read() + try: + ast.parse(content) + except SyntaxError as e: + raise SyntaxError(f"Config file {filename} has syntax error!") from e + + +def _cast_to_config(obj): + # if given a dict, return DictConfig instead + if isinstance(obj, dict): + return DictConfig(obj, flags={"allow_objects": True}) + return obj + + +_CFG_PACKAGE_NAME = "detectron2._cfg_loader" +""" +A namespace to put all imported config into. +""" + + +def _random_package_name(filename): + # generate a random package name when loading config files + return _CFG_PACKAGE_NAME + str(uuid.uuid4())[:4] + "." + os.path.basename(filename) + + +@contextmanager +def _patch_import(): + """ + Enhance relative import statements in config files, so that they: + 1. locate files purely based on relative location, regardless of packages. + e.g. you can import file without having __init__ + 2. do not cache modules globally; modifications of module states has no side effect + 3. support other storage system through PathManager, so config files can be in the cloud + 4. imported dict are turned into omegaconf.DictConfig automatically + """ + old_import = builtins.__import__ + + def find_relative_file(original_file, relative_import_path, level): + # NOTE: "from . import x" is not handled. Because then it's unclear + # if such import should produce `x` as a python module or DictConfig. + # This can be discussed further if needed. + relative_import_err = """ +Relative import of directories is not allowed within config files. +Within a config file, relative import can only import other config files. +""".replace( + "\n", " " + ) + if not len(relative_import_path): + raise ImportError(relative_import_err) + + cur_file = os.path.dirname(original_file) + for _ in range(level - 1): + cur_file = os.path.dirname(cur_file) + cur_name = relative_import_path.lstrip(".") + for part in cur_name.split("."): + cur_file = os.path.join(cur_file, part) + if not cur_file.endswith(".py"): + cur_file += ".py" + if not PathManager.isfile(cur_file): + cur_file_no_suffix = cur_file[: -len(".py")] + if PathManager.isdir(cur_file_no_suffix): + raise ImportError(f"Cannot import from {cur_file_no_suffix}." + relative_import_err) + else: + raise ImportError( + f"Cannot import name {relative_import_path} from " + f"{original_file}: {cur_file} does not exist." + ) + return cur_file + + def new_import(name, globals=None, locals=None, fromlist=(), level=0): + if ( + # Only deal with relative imports inside config files + level != 0 + and globals is not None + and (globals.get("__package__", "") or "").startswith(_CFG_PACKAGE_NAME) + ): + cur_file = find_relative_file(globals["__file__"], name, level) + _validate_py_syntax(cur_file) + spec = importlib.machinery.ModuleSpec( + _random_package_name(cur_file), None, origin=cur_file + ) + module = importlib.util.module_from_spec(spec) + module.__file__ = cur_file + with PathManager.open(cur_file) as f: + content = f.read() + exec(compile(content, cur_file, "exec"), module.__dict__) + for name in fromlist: # turn imported dict into DictConfig automatically + val = _cast_to_config(module.__dict__[name]) + module.__dict__[name] = val + return module + return old_import(name, globals, locals, fromlist=fromlist, level=level) + + builtins.__import__ = new_import + yield new_import + builtins.__import__ = old_import + + +class LazyConfig: + """ + Provide methods to save, load, and overrides an omegaconf config object + which may contain definition of lazily-constructed objects. + """ + + @staticmethod + def load_rel(filename: str, keys: Union[None, str, Tuple[str, ...]] = None): + """ + Similar to :meth:`load()`, but load path relative to the caller's + source file. + + This has the same functionality as a relative import, except that this method + accepts filename as a string, so more characters are allowed in the filename. + """ + caller_frame = inspect.stack()[1] + caller_fname = caller_frame[0].f_code.co_filename + assert caller_fname != "", "load_rel Unable to find caller" + caller_dir = os.path.dirname(caller_fname) + filename = os.path.join(caller_dir, filename) + return LazyConfig.load(filename, keys) + + @staticmethod + def load(filename: str, keys: Union[None, str, Tuple[str, ...]] = None): + """ + Load a config file. + + Args: + filename: absolute path or relative path w.r.t. the current working directory + keys: keys to load and return. If not given, return all keys + (whose values are config objects) in a dict. + """ + has_keys = keys is not None + filename = filename.replace("/./", "/") # redundant + if os.path.splitext(filename)[1] not in [".py", ".yaml", ".yml"]: + raise ValueError(f"Config file {filename} has to be a python or yaml file.") + if filename.endswith(".py"): + _validate_py_syntax(filename) + + with _patch_import(): + # Record the filename + module_namespace = { + "__file__": filename, + "__package__": _random_package_name(filename), + } + with PathManager.open(filename) as f: + content = f.read() + # Compile first with filename to: + # 1. make filename appears in stacktrace + # 2. make load_rel able to find its parent's (possibly remote) location + exec(compile(content, filename, "exec"), module_namespace) + + ret = module_namespace + else: + with PathManager.open(filename) as f: + obj = yaml.unsafe_load(f) + ret = OmegaConf.create(obj, flags={"allow_objects": True}) + + if has_keys: + if isinstance(keys, str): + return _cast_to_config(ret[keys]) + else: + return tuple(_cast_to_config(ret[a]) for a in keys) + else: + if filename.endswith(".py"): + # when not specified, only load those that are config objects + ret = DictConfig( + { + name: _cast_to_config(value) + for name, value in ret.items() + if isinstance(value, (DictConfig, ListConfig, dict)) + and not name.startswith("_") + }, + flags={"allow_objects": True}, + ) + return ret + + @staticmethod + def save(cfg, filename: str): + """ + Save a config object to a yaml file. + Note that when the config dictionary contains complex objects (e.g. lambda), + it can't be saved to yaml. In that case we will print an error and + attempt to save to a pkl file instead. + + Args: + cfg: an omegaconf config object + filename: yaml file name to save the config file + """ + logger = logging.getLogger(__name__) + try: + cfg = deepcopy(cfg) + except Exception: + pass + else: + # if it's deep-copyable, then... + def _replace_type_by_name(x): + if "_target_" in x and callable(x._target_): + try: + x._target_ = _convert_target_to_string(x._target_) + except AttributeError: + pass + + # not necessary, but makes yaml looks nicer + _visit_dict_config(cfg, _replace_type_by_name) + + save_pkl = False + try: + dict = OmegaConf.to_container( + cfg, + # Do not resolve interpolation when saving, i.e. do not turn ${a} into + # actual values when saving. + resolve=False, + # Save structures (dataclasses) in a format that can be instantiated later. + # Without this option, the type information of the dataclass will be erased. + structured_config_mode=SCMode.INSTANTIATE, + ) + dumped = yaml.dump(dict, default_flow_style=None, allow_unicode=True, width=9999) + with PathManager.open(filename, "w") as f: + f.write(dumped) + + try: + _ = yaml.unsafe_load(dumped) # test that it is loadable + except Exception: + logger.warning( + "The config contains objects that cannot serialize to a valid yaml. " + f"{filename} is human-readable but cannot be loaded." + ) + save_pkl = True + except Exception: + logger.exception("Unable to serialize the config to yaml. Error:") + save_pkl = True + + if save_pkl: + new_filename = filename + ".pkl" + try: + # retry by pickle + with PathManager.open(new_filename, "wb") as f: + cloudpickle.dump(cfg, f) + logger.warning(f"Config is saved using cloudpickle at {new_filename}.") + except Exception: + pass + + @staticmethod + def apply_overrides(cfg, overrides: List[str]): + """ + In-place override contents of cfg. + + Args: + cfg: an omegaconf config object + overrides: list of strings in the format of "a=b" to override configs. + See https://hydra.cc/docs/next/advanced/override_grammar/basic/ + for syntax. + + Returns: + the cfg object + """ + + def safe_update(cfg, key, value): + parts = key.split(".") + for idx in range(1, len(parts)): + prefix = ".".join(parts[:idx]) + v = OmegaConf.select(cfg, prefix, default=None) + if v is None: + break + if not OmegaConf.is_config(v): + raise KeyError( + f"Trying to update key {key}, but {prefix} " + f"is not a config, but has type {type(v)}." + ) + OmegaConf.update(cfg, key, value, merge=True) + + try: + from hydra.core.override_parser.overrides_parser import OverridesParser + + has_hydra = True + except ImportError: + has_hydra = False + + if has_hydra: + parser = OverridesParser.create() + overrides = parser.parse_overrides(overrides) + for o in overrides: + key = o.key_or_group + value = o.value() + if o.is_delete(): + # TODO support this + raise NotImplementedError("deletion is not yet a supported override") + safe_update(cfg, key, value) + else: + # Fallback. Does not support all the features and error checking like hydra. + for o in overrides: + key, value = o.split("=") + try: + value = ast.literal_eval(value) + except NameError: + pass + safe_update(cfg, key, value) + return cfg + + @staticmethod + def to_py(cfg, prefix: str = "cfg."): + """ + Try to convert a config object into Python-like psuedo code. + + Note that perfect conversion is not always possible. So the returned + results are mainly meant to be human-readable, and not meant to be executed. + + Args: + cfg: an omegaconf config object + prefix: root name for the resulting code (default: "cfg.") + + + Returns: + str of formatted Python code + """ + import black + + cfg = OmegaConf.to_container(cfg, resolve=True) + + def _to_str(obj, prefix=None, inside_call=False): + if prefix is None: + prefix = [] + if isinstance(obj, abc.Mapping) and "_target_" in obj: + # Dict representing a function call + target = _convert_target_to_string(obj.pop("_target_")) + args = [] + for k, v in sorted(obj.items()): + args.append(f"{k}={_to_str(v, inside_call=True)}") + args = ", ".join(args) + call = f"{target}({args})" + return "".join(prefix) + call + elif isinstance(obj, abc.Mapping) and not inside_call: + # Dict that is not inside a call is a list of top-level config objects that we + # render as one object per line with dot separated prefixes + key_list = [] + for k, v in sorted(obj.items()): + if isinstance(v, abc.Mapping) and "_target_" not in v: + key_list.append(_to_str(v, prefix=prefix + [k + "."])) + else: + key = "".join(prefix) + k + key_list.append(f"{key}={_to_str(v)}") + return "\n".join(key_list) + elif isinstance(obj, abc.Mapping): + # Dict that is inside a call is rendered as a regular dict + return ( + "{" + + ",".join( + f"{repr(k)}: {_to_str(v, inside_call=inside_call)}" + for k, v in sorted(obj.items()) + ) + + "}" + ) + elif isinstance(obj, list): + return "[" + ",".join(_to_str(x, inside_call=inside_call) for x in obj) + "]" + else: + return repr(obj) + + py_str = _to_str(cfg, prefix=[prefix]) + try: + return black.format_str(py_str, mode=black.Mode()) + except black.InvalidInput: + return py_str \ No newline at end of file diff --git a/lazy_config/utils/file_io.py b/lazy_config/utils/file_io.py new file mode 100644 index 0000000..46c53d5 --- /dev/null +++ b/lazy_config/utils/file_io.py @@ -0,0 +1,54 @@ +# -*- coding: utf-8 -*- +# Original Copyright (c) Facebook, Inc. and its affiliates. +# This file has been modified by Yuyao Huang (C) 2025. +# SPDX-License-Identifier: Apache-2.0 +# +# This file is based on code from Detectron2 (https://github.com/facebookresearch/detectron2) +# and is licensed under the Apache License, Version 2.0. +# You may obtain a copy of the License at (http://www.apache.org/licenses/LICENSE-2.0). +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from iopath.common.file_io import HTTPURLHandler, OneDrivePathHandler, PathHandler +from iopath.common.file_io import PathManager as PathManagerBase + +__all__ = ["PathManager", "PathHandler"] + + +PathManager = PathManagerBase() +""" +This is a detectron2 project-specific PathManager. +We try to stay away from global PathManager in fvcore as it +introduces potential conflicts among other libraries. +""" + + +class Detectron2Handler(PathHandler): + """ + Resolve anything that's hosted under detectron2's namespace. + """ + + PREFIX = "detectron2://" + S3_DETECTRON2_PREFIX = "https://dl.fbaipublicfiles.com/detectron2/" + + def _get_supported_prefixes(self): + return [self.PREFIX] + + def _get_local_path(self, path, **kwargs): + name = path[len(self.PREFIX) :] + return PathManager.get_local_path(self.S3_DETECTRON2_PREFIX + name, **kwargs) + + def _open(self, path, mode="r", **kwargs): + return PathManager.open( + self.S3_DETECTRON2_PREFIX + path[len(self.PREFIX) :], mode, **kwargs + ) + + +PathManager.register_handler(HTTPURLHandler()) +PathManager.register_handler(OneDrivePathHandler()) +PathManager.register_handler(Detectron2Handler()) diff --git a/lazy_config/utils/registry.py b/lazy_config/utils/registry.py new file mode 100644 index 0000000..6848305 --- /dev/null +++ b/lazy_config/utils/registry.py @@ -0,0 +1,73 @@ +# -*- coding: utf-8 -*- +# Original Copyright (c) Facebook, Inc. and its affiliates. +# This file has been modified by Yuyao Huang (C) 2025. +# SPDX-License-Identifier: Apache-2.0 +# +# This file is based on code from Detectron2 (https://github.com/facebookresearch/detectron2) +# and is licensed under the Apache License, Version 2.0. +# You may obtain a copy of the License at (http://www.apache.org/licenses/LICENSE-2.0). +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from typing import Any +import pydoc + +""" +`locate` maps a string (typically found in config files) to callable objects. +""" + +__all__ = ["locate"] + + +def _convert_target_to_string(t: Any) -> str: + """ + Inverse of ``locate()``. + + Args: + t: any object with ``__module__`` and ``__qualname__`` + """ + module, qualname = t.__module__, t.__qualname__ + + # Compress the path to this object, e.g. ``module.submodule._impl.class`` + # may become ``module.submodule.class``, if the later also resolves to the same + # object. This simplifies the string, and also is less affected by moving the + # class implementation. + module_parts = module.split(".") + for k in range(1, len(module_parts)): + prefix = ".".join(module_parts[:k]) + candidate = f"{prefix}.{qualname}" + try: + if locate(candidate) is t: + return candidate + except ImportError: + pass + return f"{module}.{qualname}" + + +def locate(name: str) -> Any: + """ + Locate and return an object ``x`` using an input string ``{x.__module__}.{x.__qualname__}``, + such as "module.submodule.class_name". + + Raise Exception if it cannot be found. + """ + obj = pydoc.locate(name) + + # Some cases (e.g. torch.optim.sgd.SGD) not handled correctly + # by pydoc.locate. Try a private function from hydra. + if obj is None: + try: + # from hydra.utils import get_method - will print many errors + from hydra.utils import _locate + except ImportError as e: + raise ImportError( + f"Cannot dynamically locate object {name}!") from e + else: + obj = _locate(name) # it raises if fails + + return obj diff --git a/projects/.gitkeep b/projects/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/projects/lazy_config_demo/config.py b/projects/lazy_config_demo/config.py new file mode 100644 index 0000000..926b580 --- /dev/null +++ b/projects/lazy_config_demo/config.py @@ -0,0 +1,9 @@ +from lazy_config.utils.config import LazyCall as L +from projects.lazy_config_demo.model import SimpleModule + + +MODEL = L(SimpleModule)( + in_features=128, + out_features=256, + activation="sigmoid" +) \ No newline at end of file diff --git a/projects/lazy_config_demo/main.py b/projects/lazy_config_demo/main.py new file mode 100644 index 0000000..c212890 --- /dev/null +++ b/projects/lazy_config_demo/main.py @@ -0,0 +1,11 @@ +from lazy_config.utils.config import parse_args_and_configs, instantiate + + +def main(): + cfg = parse_args_and_configs() + model = instantiate(cfg.MODEL) + print(model) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/projects/lazy_config_demo/model.py b/projects/lazy_config_demo/model.py new file mode 100644 index 0000000..46b21be --- /dev/null +++ b/projects/lazy_config_demo/model.py @@ -0,0 +1,14 @@ +class SimpleModule: + + def __init__(self, + in_features: int, + out_features: int, + activation: str = "relu"): + self.in_features = in_features + self.out_features = out_features + self.activation = activation + print( + f"SimpleModule created: {in_features} -> {out_features}, activation: {activation}" + ) + + diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..fae3296 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,38 @@ +[build-system] +requires = ["setuptools>=65.5.0", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "lazy_config" # pip install +version = "0.1.0" +description = "..." +readme = "README.md" # 可选, 如果你有 README.md 文件 +requires-python = ">=3.10" +license = {text = "Apache-2.0"} # 或者你的许可证 +authors = [ + {name = "Yuyao Huang (Sam)", email = "huangyuyao@outlook.com"}, +] +dependencies = [ + "cloudpickle", + "omegaconf", + "iopath", + "hydra-core", +] + +[tool.setuptools] +py-modules = ["lazy_config"] + +# 包含非 Python 文件 +# [tool.setuptools.package-data] +# "package.path" = ["*.json"] + +# 命令行程序入口 +# [project.scripts] +# script-name = "module.path:function" + +[project.optional-dependencies] +dev = [ + "pytest", + "pytest-cov", + "pytest-mock", +]