Compare commits
2 Commits
8bc760ad8e
...
a86e38b408
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
a86e38b408 | ||
|
|
cbec53af34 |
18
.vscode/settings.json
vendored
Normal file
18
.vscode/settings.json
vendored
Normal file
@ -0,0 +1,18 @@
|
|||||||
|
{
|
||||||
|
"workbench.colorCustomizations": {
|
||||||
|
"titleBar.activeBackground": "#ffb5d5",
|
||||||
|
"titleBar.activeForeground": "#000000",
|
||||||
|
"titleBar.inactiveBackground": "#ffb5d5",
|
||||||
|
"titleBar.inactiveForeground": "#000000",
|
||||||
|
"titleBar.border": "#ffb5d5",
|
||||||
|
"activityBar.background": "#ffb5d5",
|
||||||
|
"activityBar.foreground": "#000000",
|
||||||
|
"statusBar.background": "#ffb5d5",
|
||||||
|
"statusBar.foreground": "#000000",
|
||||||
|
"statusBar.debuggingBackground": "#ffb5d5",
|
||||||
|
"statusBar.debuggingForeground": "#000000",
|
||||||
|
"tab.activeBorder": "#ffb5d5",
|
||||||
|
"iLoveWorkSpaceColors": true,
|
||||||
|
"iLoveWorkSpaceRandom": false
|
||||||
|
}
|
||||||
|
}
|
||||||
181
LICENSE
Normal file
181
LICENSE
Normal file
@ -0,0 +1,181 @@
|
|||||||
|
Apache License
|
||||||
|
Version 2.0, January 2004
|
||||||
|
http://www.apache.org/licenses/
|
||||||
|
|
||||||
|
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||||
|
|
||||||
|
1. Definitions.
|
||||||
|
|
||||||
|
"License" shall mean the terms and conditions for use, reproduction,
|
||||||
|
and distribution as defined by Sections 1 through 9 of this document.
|
||||||
|
|
||||||
|
"Licensor" shall mean the copyright owner or entity authorized by
|
||||||
|
the copyright owner that is granting the License.
|
||||||
|
|
||||||
|
"Legal Entity" shall mean the union of the acting entity and all
|
||||||
|
other entities that control, are controlled by, or are under common
|
||||||
|
control with that entity. For the purposes of this definition,
|
||||||
|
"control" means (i) the power, direct or indirect, to cause the
|
||||||
|
direction or management of such entity, whether by contract or
|
||||||
|
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||||
|
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||||
|
|
||||||
|
"You" (or "Your") shall mean an individual or Legal Entity
|
||||||
|
exercising permissions granted by this License.
|
||||||
|
|
||||||
|
"Source" form shall mean the preferred form for making modifications,
|
||||||
|
including but not limited to software source code, documentation
|
||||||
|
source, and configuration files.
|
||||||
|
|
||||||
|
"Object" form shall mean any form resulting from mechanical
|
||||||
|
transformation or translation of a Source form, including but
|
||||||
|
not limited to compiled object code, generated documentation,
|
||||||
|
and conversions to other media types.
|
||||||
|
|
||||||
|
"Work" shall mean the work of authorship, whether in Source or
|
||||||
|
Object form, made available under the License, as indicated by a
|
||||||
|
copyright notice that is included in or attached to the Work
|
||||||
|
(an example is provided in the Appendix below).
|
||||||
|
|
||||||
|
"Derivative Works" shall mean any work, whether in Source or Object
|
||||||
|
form, that is based on (or derived from) the Work and for which the
|
||||||
|
copyrighted components of the original Work have been recombined,
|
||||||
|
rearranged, modified, or otherwise modified.
|
||||||
|
|
||||||
|
"Contribution" shall mean any work of authorship, including
|
||||||
|
the original version of the Work and any modifications or additions
|
||||||
|
to that Work or Derivative Works thereof, that is intentionally
|
||||||
|
submitted to Licensor for inclusion in the Work by the copyright owner
|
||||||
|
or by an individual or Legal Entity authorized to submit on behalf of
|
||||||
|
the copyright owner.
|
||||||
|
|
||||||
|
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||||
|
on behalf of whom a Contribution has been received by Licensor and
|
||||||
|
subsequently incorporated within the Work.
|
||||||
|
|
||||||
|
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||||
|
this License, each Contributor hereby grants to You a perpetual,
|
||||||
|
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||||
|
copyright license to reproduce, prepare Derivative Works of,
|
||||||
|
publicly display, publicly perform, sublicense, and distribute the
|
||||||
|
Work and such Derivative Works in Source or Object form.
|
||||||
|
|
||||||
|
3. Grant of Patent License. Subject to the terms and conditions of
|
||||||
|
this License, each Contributor hereby grants to You a perpetual,
|
||||||
|
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||||
|
(except as stated in this section) patent license to make, have made,
|
||||||
|
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||||
|
where such license applies only to those patent claims licensable
|
||||||
|
by such Contributor that are necessarily infringed by their
|
||||||
|
Contribution(s) alone or by combination of their Contribution(s)
|
||||||
|
with the Work to which such Contribution(s) was submitted. If You
|
||||||
|
institute patent litigation against any entity (including a
|
||||||
|
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||||
|
or a Contribution incorporated within the Work constitutes direct
|
||||||
|
or contributory patent infringement, then any patent licenses
|
||||||
|
granted to You under this License for that Work shall terminate
|
||||||
|
as of the date such litigation is filed.
|
||||||
|
|
||||||
|
4. Redistribution. You may reproduce and distribute copies of the
|
||||||
|
Work or Derivative Works thereof in any medium, with or without
|
||||||
|
modifications, and in Source or Object form, provided that You
|
||||||
|
meet the following conditions:
|
||||||
|
|
||||||
|
(a) You must give any other recipients of the Work or Derivative Works
|
||||||
|
a copy of this License; and
|
||||||
|
|
||||||
|
(b) You must cause any modified files to carry prominent notices stating
|
||||||
|
that You changed the files; and
|
||||||
|
|
||||||
|
(c) You must retain, in the Source form of any Derivative Works that You
|
||||||
|
distribute, all copyright, patent, trademark, and attribution notices
|
||||||
|
from the Source form of the Work, excluding those notices that do not
|
||||||
|
pertain to any part of the Derivative Works; and
|
||||||
|
|
||||||
|
(d) If the Work includes a "NOTICE" text file as part of its
|
||||||
|
distribution, then any Derivative Works that You distribute must
|
||||||
|
include a readable copy of the attribution notices contained
|
||||||
|
within such NOTICE file, excluding those notices that do not
|
||||||
|
pertain to any part of the Derivative Works, in at least one
|
||||||
|
of the following places: within a NOTICE text file distributed
|
||||||
|
as part of the Derivative Works; within the Source form or
|
||||||
|
documentation, if provided along with the Derivative Works; or,
|
||||||
|
within a display generated by the Derivative Works, if and
|
||||||
|
where such third-party notices normally appear. The contents of the
|
||||||
|
NOTICE file are for informational purposes only and do not modify
|
||||||
|
the License. You may add Your own attribution notices within
|
||||||
|
Derivative Works that You distribute, alongside, or as an addendum to
|
||||||
|
the NOTICE text from the Work, provided that such additional
|
||||||
|
attribution notices cannot be construed as modifying the License.
|
||||||
|
|
||||||
|
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||||
|
any Contribution intentionally submitted for inclusion in the Work
|
||||||
|
by You to the Licensor shall be under the terms and conditions of
|
||||||
|
this License, without any additional terms or conditions.
|
||||||
|
Notwithstanding the above, nothing herein shall supersede or modify
|
||||||
|
the terms of any separate license agreement you may have executed
|
||||||
|
with Licensor regarding such Contributions.
|
||||||
|
|
||||||
|
6. Trademarks. This License does not grant permission to use the trade
|
||||||
|
names, trademarks, service marks, or product names of the Licensor,
|
||||||
|
except as required for reasonable and customary use in describing the
|
||||||
|
origin of the Work and reproducing the content of the NOTICE file.
|
||||||
|
|
||||||
|
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||||
|
agreed to in writing, Licensor provides the Work (and each
|
||||||
|
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||||
|
implied, including, without limitation, any warranties or conditions
|
||||||
|
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||||
|
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||||
|
appropriateness of using or redistributing the Work and assume any
|
||||||
|
risks associated with Your exercise of permissions under this License.
|
||||||
|
|
||||||
|
8. Limitation of Liability. In no event and under no legal theory,
|
||||||
|
whether in tort (including negligence), contract, or otherwise,
|
||||||
|
unless required by applicable law (such as deliberate and grossly
|
||||||
|
negligent acts) or agreed to in writing, shall any Contributor be
|
||||||
|
liable to You for damages, including any direct, indirect, special,
|
||||||
|
incidental, or consequential damages of any character arising as a
|
||||||
|
result of this License or out of the use or inability to use the
|
||||||
|
Work (including but not limited to damages for loss of goodwill,
|
||||||
|
work stoppage, computer failure or malfunction, or any and all
|
||||||
|
other commercial damages or losses), even if such Contributor
|
||||||
|
has been advised of the possibility of such damages.
|
||||||
|
|
||||||
|
9. Accepting Warranty or Additional Liability. While redistributing
|
||||||
|
the Work or Derivative Works thereof, You may choose to offer,
|
||||||
|
and charge a fee for, acceptance of support, warranty, indemnity,
|
||||||
|
or other liability obligations and/or rights consistent with this
|
||||||
|
License. However, in accepting such obligations, You may act only
|
||||||
|
on Your own behalf and on Your sole responsibility, not on behalf
|
||||||
|
of any other Contributor, and only if You agree to indemnify,
|
||||||
|
defend, and hold each Contributor harmless for any liability
|
||||||
|
incurred by, or claims asserted against, such Contributor by reason
|
||||||
|
of your accepting any such warranty or additional liability.
|
||||||
|
|
||||||
|
END OF TERMS AND CONDITIONS
|
||||||
|
|
||||||
|
APPENDIX: How to apply the Apache License to your work.
|
||||||
|
|
||||||
|
To apply the Apache License to your work, attach the following
|
||||||
|
boilerplate notice, with the fields enclosed by brackets "[]" replaced
|
||||||
|
with your own identifying information. (Don't include the brackets!)
|
||||||
|
The text should be enclosed in the appropriate comment syntax for
|
||||||
|
the file format. We also recommend that a file or class name and
|
||||||
|
description of purpose be included on the same "printed page" as the
|
||||||
|
copyright notice for easier identification within third-party archives.
|
||||||
|
|
||||||
|
Copyright 2025 Yuyao Huang
|
||||||
|
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
you may not use this file except in compliance with the License.
|
||||||
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
34
README.md
34
README.md
@ -1,3 +1,35 @@
|
|||||||
# lazy_config
|
# lazy_config
|
||||||
|
|
||||||
剥离 detectron2 的 LazyConfig 功能,用于通用任务的项目模板。
|
剥离 detectron2 的 LazyConfig 功能,用于通用任务的项目模板。
|
||||||
|
|
||||||
|
## How to create your own package based on this package.
|
||||||
|
|
||||||
|
Use the following command to create a new package based on this package:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
python init.py your-package-name
|
||||||
|
```
|
||||||
|
|
||||||
|
It will replace the current package name (`lazy_config`) with your own package name in the code base.
|
||||||
|
|
||||||
|
However, you need to modify the author name and email in `pyproject.toml`.
|
||||||
|
|
||||||
|
Then you should use `pip install -e .[dev]` to install the package in editable mode.
|
||||||
|
|
||||||
|
After the project is created, you can delete the `init.py` file and start working on your project.
|
||||||
|
|
||||||
|
## Usage
|
||||||
|
|
||||||
|
See the example in `projects/lazy_config_demo/main.py` and run it with the following command:
|
||||||
|
|
||||||
|
```
|
||||||
|
PYTHONPATH=. python projects/lazy_config_demo/main.py projects/lazy_config_demo/config.py MODEL.in_features=10
|
||||||
|
```
|
||||||
|
|
||||||
|
## Principles
|
||||||
|
|
||||||
|
- `LazyCall` (or `L`): 这是让你能够以惰性方式定义对象构建的语法糖。
|
||||||
|
|
||||||
|
- `instantiate()` function: 这是将惰性配置(带有 _target_ 的 omegaconf 对象)转换为实际 Python 对象的关键函数。它会处理递归实例化和参数传递。
|
||||||
|
|
||||||
|
- `parse_args_and_configs()` function: 它允许你读取命令行参数和配置文件,并将它们合并成一个 omegaconf 对象。
|
||||||
269
init.py
Normal file
269
init.py
Normal file
@ -0,0 +1,269 @@
|
|||||||
|
import argparse
|
||||||
|
import keyword
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import shutil
|
||||||
|
import sys
|
||||||
|
|
||||||
|
def main():
|
||||||
|
args = argparse.ArgumentParser()
|
||||||
|
args.add_argument("new_package_name")
|
||||||
|
args.add_argument("--dry-run", action="store_true")
|
||||||
|
args = args.parse_args()
|
||||||
|
|
||||||
|
is_valid_name, reason = is_valid_python_package_name(args.new_package_name)
|
||||||
|
|
||||||
|
assert is_valid_name, reason
|
||||||
|
|
||||||
|
replace_in_multiple_paths([
|
||||||
|
"lazy_config",
|
||||||
|
"projects",
|
||||||
|
"README.md",
|
||||||
|
"pyproject.toml"
|
||||||
|
],
|
||||||
|
old_word="lazy_config",
|
||||||
|
new_word=args.new_package_name,
|
||||||
|
file_extensions=('.py', '.md', '.toml'),
|
||||||
|
dry_run=args.dry_run
|
||||||
|
)
|
||||||
|
|
||||||
|
if not args.dry_run:
|
||||||
|
shutil.move("lazy_config", args.new_package_name)
|
||||||
|
|
||||||
|
|
||||||
|
def is_valid_python_package_name(name: str) -> tuple[bool, str]:
|
||||||
|
"""
|
||||||
|
检查一个字符串是否是合格的 Python 包/模块名。
|
||||||
|
|
||||||
|
这个函数综合考虑了 Python 语法规则和一些常见的最佳实践。
|
||||||
|
|
||||||
|
Args:
|
||||||
|
name (str): 要检查的字符串。
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
tuple[bool, str]: 一个元组,第一个元素是布尔值(True 表示合格,False 表示不合格),
|
||||||
|
第二个元素是不合格的原因或“Valid.”。
|
||||||
|
"""
|
||||||
|
if not name:
|
||||||
|
return False, "包/模块名不能为空。"
|
||||||
|
|
||||||
|
# 将包名按点分割成各个组件
|
||||||
|
segments = name.split('.')
|
||||||
|
|
||||||
|
for segment in segments:
|
||||||
|
if not segment:
|
||||||
|
return False, f"包/模块名 '{name}' 包含空组件 (例如 'a..b' 或开头/结尾是点)。"
|
||||||
|
|
||||||
|
# 规则1: 每个组件都必须是有效的 Python 标识符。
|
||||||
|
# isidentifier() 检查了:
|
||||||
|
# - 以字母或下划线开头 (不能以数字开头)
|
||||||
|
# - 仅包含字母、数字或下划线
|
||||||
|
# - 非空 (虽然上面单独检查了)
|
||||||
|
if not segment.isidentifier():
|
||||||
|
return False, f"组件 '{segment}' 在 '{name}' 中不是一个有效的 Python 标识符。"
|
||||||
|
|
||||||
|
# 规则2: 每个组件都不能是 Python 关键字。
|
||||||
|
if keyword.iskeyword(segment):
|
||||||
|
return False, f"组件 '{segment}' 在 '{name}' 中是一个 Python 关键字。"
|
||||||
|
|
||||||
|
# 规则3 (强烈建议): 避免与内置模块名冲突。
|
||||||
|
# 尽管技术上可以导入,但这会导致混淆和潜在的导入问题。
|
||||||
|
if segment in sys.builtin_module_names:
|
||||||
|
return False, f"组件 '{segment}' 在 '{name}' 中与一个内置的 Python 模块名冲突。"
|
||||||
|
|
||||||
|
# 仅作为PEP 8风格的额外检查,不影响“合格性”,如果需要严格的PEP8,可以将其改为返回False
|
||||||
|
# if not segment.islower() and not segment.startswith('__') and not segment.endswith('__'):
|
||||||
|
# # 允许像 __init__ 这样的特殊名称不是小写
|
||||||
|
# # 对于普通的模块/包名,PEP 8 推荐全小写
|
||||||
|
# # return False, f"组件 '{segment}' 在 '{name}' 中不符合 PEP 8 的全小写命名约定。"
|
||||||
|
# if segment.startswith('_') and not segment == '__init__':
|
||||||
|
# # PEP 8 鼓励避免普通模块/包名的前导下划线
|
||||||
|
# # return False, f"组件 '{segment}' 在 '{name}' 中不符合 PEP 8 的前导下划线约定。"
|
||||||
|
# if segment.endswith('_'):
|
||||||
|
# # PEP 8 鼓励避免普通模块/包名的尾随下划线
|
||||||
|
# # return False, f"组件 '{segment}' 在 '{name}' 中不符合 PEP 8 的尾随下划线约定。"
|
||||||
|
pass
|
||||||
|
|
||||||
|
return True, "Valid."
|
||||||
|
|
||||||
|
|
||||||
|
def replace_in_single_file(file_path: str, old_word: str, new_word: str,
|
||||||
|
encoding: str = 'utf-8', dry_run: bool = False) -> tuple[bool, list[str]]:
|
||||||
|
"""
|
||||||
|
在单个文件中替换特定全字匹配的字符串,并记录每一处替换内容。
|
||||||
|
支持 dry_run 模式。
|
||||||
|
|
||||||
|
Args:
|
||||||
|
file_path (str): 文件的完整路径。
|
||||||
|
old_word (str): 要被替换的全字匹配字符串。
|
||||||
|
new_word (str): 替换成的字符串。
|
||||||
|
encoding (str): 读取和写入文件时使用的编码。
|
||||||
|
dry_run (bool): 如果为 True, 则只记录将要进行的替换,不实际修改文件。
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
tuple[bool, list[str]]:
|
||||||
|
- bool: 是否存在匹配并进行了(或将要进行)替换。
|
||||||
|
- list[str]: 记录了所有替换操作(或模拟操作)的文本日志。
|
||||||
|
每个日志条目包含行号和高亮显示的代码行。
|
||||||
|
"""
|
||||||
|
replacement_logs = []
|
||||||
|
|
||||||
|
try:
|
||||||
|
if not os.path.exists(file_path):
|
||||||
|
replacement_logs.append(f" Error: File not found: {file_path}")
|
||||||
|
return False, replacement_logs
|
||||||
|
|
||||||
|
with open(file_path, 'r', encoding=encoding) as f:
|
||||||
|
content = f.read()
|
||||||
|
|
||||||
|
pattern = r"\b" + re.escape(old_word) + r"\b"
|
||||||
|
matches = list(re.finditer(pattern, content))
|
||||||
|
|
||||||
|
if not matches:
|
||||||
|
return False, [] # 没有匹配,直接返回
|
||||||
|
|
||||||
|
def repl_func(match):
|
||||||
|
match_start = match.start()
|
||||||
|
line_num = content[:match_start].count('\n') + 1
|
||||||
|
|
||||||
|
line_start_idx = content.rfind('\n', 0, match_start) + 1
|
||||||
|
line_end_idx = content.find('\n', match.end())
|
||||||
|
if line_end_idx == -1:
|
||||||
|
line_end_idx = len(content)
|
||||||
|
|
||||||
|
original_line = content[line_start_idx:line_end_idx].strip()
|
||||||
|
|
||||||
|
relative_match_start = match_start - line_start_idx
|
||||||
|
relative_match_end = match.end() - line_start_idx
|
||||||
|
highlighted_line = (
|
||||||
|
original_line[:relative_match_start] +
|
||||||
|
f"**{original_line[relative_match_start:relative_match_end]}**" +
|
||||||
|
original_line[relative_match_end:]
|
||||||
|
)
|
||||||
|
|
||||||
|
replacement_logs.append(f" L{line_num:<4}: '{highlighted_line}' -> '{new_word}'")
|
||||||
|
|
||||||
|
return new_word
|
||||||
|
|
||||||
|
new_content = re.sub(pattern, repl_func, content)
|
||||||
|
|
||||||
|
if not dry_run:
|
||||||
|
with open(file_path, 'w', encoding=encoding) as f:
|
||||||
|
f.write(new_content)
|
||||||
|
|
||||||
|
return True, replacement_logs
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
replacement_logs.append(f" Error processing {file_path}: {e}")
|
||||||
|
return False, replacement_logs
|
||||||
|
|
||||||
|
# --- 新增的辅助函数 (保持不变) ---
|
||||||
|
def _process_a_file(file_path: str, old_word: str, new_word: str,
|
||||||
|
file_extensions: tuple[str, ...], # 变更:现在是元组
|
||||||
|
encoding: str, dry_run: bool, stats: dict):
|
||||||
|
"""
|
||||||
|
辅助函数,封装了对单个文件的处理逻辑并更新统计信息。
|
||||||
|
"""
|
||||||
|
# 检查文件扩展名是否符合要求
|
||||||
|
# 如果 file_extensions 是空元组,表示不限制后缀,总是匹配
|
||||||
|
if file_extensions and not file_path.endswith(file_extensions): # endswith 可以直接接受元组
|
||||||
|
stats['skipped'] += 1
|
||||||
|
return
|
||||||
|
|
||||||
|
found_match, logs = replace_in_single_file(
|
||||||
|
file_path, old_word, new_word, encoding, dry_run
|
||||||
|
)
|
||||||
|
|
||||||
|
if logs:
|
||||||
|
print(f"File: {file_path}")
|
||||||
|
for log_entry in logs:
|
||||||
|
print(log_entry)
|
||||||
|
print("-" * (len(file_path) + 6))
|
||||||
|
|
||||||
|
if found_match:
|
||||||
|
if not dry_run:
|
||||||
|
print(f" ✔ Modified: {file_path}\n")
|
||||||
|
else:
|
||||||
|
print(f" ✔ Will modify: {file_path}\n")
|
||||||
|
stats['modified'] += 1
|
||||||
|
elif not logs and os.path.exists(file_path):
|
||||||
|
stats['skipped'] += 1
|
||||||
|
else:
|
||||||
|
stats['errors'] += 1
|
||||||
|
|
||||||
|
|
||||||
|
# --- 修改后的主协调函数 ---
|
||||||
|
def replace_in_multiple_paths(paths_to_process: list[str], old_word: str, new_word: str,
|
||||||
|
file_extensions: str | tuple[str, ...] = (), # 变更:接受字符串或元组
|
||||||
|
encoding: str = 'utf-8',
|
||||||
|
dry_run: bool = False):
|
||||||
|
"""
|
||||||
|
根据提供的文件和文件夹路径列表,替换所有全字匹配的字符串。
|
||||||
|
支持 dry_run 模式,并允许多种文件后缀过滤。
|
||||||
|
|
||||||
|
Args:
|
||||||
|
paths_to_process (list[str]): 包含要处理的文件和/或文件夹路径的列表。
|
||||||
|
old_word (str): 要被替换的全字匹配字符串。
|
||||||
|
new_word (str): 替换成的字符串。
|
||||||
|
file_extensions (str | tuple[str, ...]):
|
||||||
|
要处理的文件扩展名。可以是单个字符串 (例如 ".py"),
|
||||||
|
也可以是字符串元组 (例如 (".py", ".txt"))。
|
||||||
|
默认 (空元组) 表示处理所有文件扩展名。
|
||||||
|
encoding (str): 读取和写入文件时使用的编码。
|
||||||
|
dry_run (bool): 如果为 True, 则只打印将要进行的替换,不实际修改文件。
|
||||||
|
"""
|
||||||
|
# Normalize file_extensions to always be a tuple, empty if no filter
|
||||||
|
if isinstance(file_extensions, str):
|
||||||
|
if file_extensions == "":
|
||||||
|
normalized_file_extensions = ()
|
||||||
|
else:
|
||||||
|
normalized_file_extensions = (file_extensions,)
|
||||||
|
else:
|
||||||
|
normalized_file_extensions = file_extensions
|
||||||
|
|
||||||
|
action_verb = "Simulating replacement" if dry_run else "Performing replacement"
|
||||||
|
print(f"--- {action_verb} ---")
|
||||||
|
print(f"Searching for full word '{old_word}'")
|
||||||
|
print(f"Replacing with '{new_word}'")
|
||||||
|
print(f"In paths: {paths_to_process}")
|
||||||
|
if normalized_file_extensions:
|
||||||
|
print(f"Filtered by extensions: {','.join(normalized_file_extensions)}")
|
||||||
|
else:
|
||||||
|
print(f"No extension filter (processing all files).")
|
||||||
|
|
||||||
|
if dry_run:
|
||||||
|
print(" (DRY RUN: No files will be modified)")
|
||||||
|
print("-" * 60)
|
||||||
|
|
||||||
|
stats = {'modified': 0, 'skipped': 0, 'errors': 0, 'invalid_paths': 0}
|
||||||
|
|
||||||
|
for path in paths_to_process:
|
||||||
|
if not os.path.exists(path):
|
||||||
|
print(f"‼ Warning: Path does not exist and will be skipped: '{path}'\n")
|
||||||
|
stats['invalid_paths'] += 1
|
||||||
|
continue
|
||||||
|
|
||||||
|
if os.path.isfile(path):
|
||||||
|
_process_a_file(path, old_word, new_word, normalized_file_extensions, encoding, dry_run, stats)
|
||||||
|
elif os.path.isdir(path):
|
||||||
|
# print(f"Processing directory: '{path}'\n")
|
||||||
|
for root, _, files in os.walk(path):
|
||||||
|
for file_name in files:
|
||||||
|
file_path = os.path.join(root, file_name)
|
||||||
|
_process_a_file(file_path, old_word, new_word, normalized_file_extensions, encoding, dry_run, stats)
|
||||||
|
else:
|
||||||
|
print(f"‼ Warning: Skipping '{path}' (not a file or directory).\n")
|
||||||
|
stats['invalid_paths'] += 1
|
||||||
|
|
||||||
|
print("\n" + "=" * 60)
|
||||||
|
print("--- Summary ---")
|
||||||
|
print(f"Total files checked: {stats['modified'] + stats['skipped'] + stats['errors']}")
|
||||||
|
print(f"Files to be modified/modified: {stats['modified']}")
|
||||||
|
print(f"Files skipped (word not found or extension mismatch): {stats['skipped']}")
|
||||||
|
print(f"Files with errors: {stats['errors']}")
|
||||||
|
print(f"Invalid paths provided: {stats['invalid_paths']}")
|
||||||
|
print("=" * 60)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
0
lazy_config/__init__.py
Normal file
0
lazy_config/__init__.py
Normal file
0
lazy_config/utils/__init__.py
Normal file
0
lazy_config/utils/__init__.py
Normal file
10
lazy_config/utils/config/__init__.py
Normal file
10
lazy_config/utils/config/__init__.py
Normal file
@ -0,0 +1,10 @@
|
|||||||
|
from .instantiate import instantiate
|
||||||
|
from .lazy import LazyCall, LazyConfig
|
||||||
|
from .argparser import parse_args_and_configs
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"instantiate",
|
||||||
|
"LazyCall",
|
||||||
|
"LazyConfig",
|
||||||
|
"parse_args_and_configs",
|
||||||
|
]
|
||||||
28
lazy_config/utils/config/argparser.py
Normal file
28
lazy_config/utils/config/argparser.py
Normal file
@ -0,0 +1,28 @@
|
|||||||
|
import argparse
|
||||||
|
import os
|
||||||
|
from .lazy import LazyConfig
|
||||||
|
|
||||||
|
|
||||||
|
def parse_args_and_configs(program_description: str = ""):
|
||||||
|
parser = argparse.ArgumentParser(description=program_description)
|
||||||
|
parser.add_argument(
|
||||||
|
"config_file",
|
||||||
|
metavar="FILE",
|
||||||
|
help="path to config file",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"opts",
|
||||||
|
help="Modify config options using the command-line 'KEY VALUE' pairs",
|
||||||
|
default=None,
|
||||||
|
nargs=argparse.REMAINDER,
|
||||||
|
)
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
if not os.path.exists(args.config_file):
|
||||||
|
raise FileNotFoundError(f"Config file not found: {args.config_file}")
|
||||||
|
cfg = LazyConfig.load(args.config_file)
|
||||||
|
|
||||||
|
if args.opts:
|
||||||
|
cfg.merge_with_dotlist(args.opts)
|
||||||
|
|
||||||
|
return cfg
|
||||||
101
lazy_config/utils/config/instantiate.py
Normal file
101
lazy_config/utils/config/instantiate.py
Normal file
@ -0,0 +1,101 @@
|
|||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
# Original Copyright (c) Facebook, Inc. and its affiliates.
|
||||||
|
# This file has been modified by Yuyao Huang (C) 2025.
|
||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
#
|
||||||
|
# This file is based on code from Detectron2 (https://github.com/facebookresearch/detectron2)
|
||||||
|
# and is licensed under the Apache License, Version 2.0.
|
||||||
|
# You may obtain a copy of the License at (http://www.apache.org/licenses/LICENSE-2.0).
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
|
||||||
|
import collections.abc as abc
|
||||||
|
import dataclasses
|
||||||
|
import logging
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from lazy_config.utils.registry import _convert_target_to_string, locate
|
||||||
|
|
||||||
|
__all__ = ["dump_dataclass", "instantiate"]
|
||||||
|
|
||||||
|
|
||||||
|
def dump_dataclass(obj: Any):
|
||||||
|
"""
|
||||||
|
Dump a dataclass recursively into a dict that can be later instantiated.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
obj: a dataclass object
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
dict
|
||||||
|
"""
|
||||||
|
assert dataclasses.is_dataclass(obj) and not isinstance(
|
||||||
|
obj, type
|
||||||
|
), "dump_dataclass() requires an instance of a dataclass."
|
||||||
|
ret = {"_target_": _convert_target_to_string(type(obj))}
|
||||||
|
for f in dataclasses.fields(obj):
|
||||||
|
v = getattr(obj, f.name)
|
||||||
|
if dataclasses.is_dataclass(v):
|
||||||
|
v = dump_dataclass(v)
|
||||||
|
if isinstance(v, (list, tuple)):
|
||||||
|
v = [dump_dataclass(x) if dataclasses.is_dataclass(x) else x for x in v]
|
||||||
|
ret[f.name] = v
|
||||||
|
return ret
|
||||||
|
|
||||||
|
|
||||||
|
def instantiate(cfg):
|
||||||
|
"""
|
||||||
|
Recursively instantiate objects defined in dictionaries by
|
||||||
|
"_target_" and arguments.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
cfg: a dict-like object with "_target_" that defines the caller, and
|
||||||
|
other keys that define the arguments
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
object instantiated by cfg
|
||||||
|
"""
|
||||||
|
from omegaconf import ListConfig, DictConfig, OmegaConf
|
||||||
|
|
||||||
|
if isinstance(cfg, ListConfig):
|
||||||
|
lst = [instantiate(x) for x in cfg]
|
||||||
|
return ListConfig(lst, flags={"allow_objects": True})
|
||||||
|
if isinstance(cfg, list):
|
||||||
|
# Specialize for list, because many classes take
|
||||||
|
# list[objects] as arguments, such as ResNet, DatasetMapper
|
||||||
|
return [instantiate(x) for x in cfg]
|
||||||
|
|
||||||
|
# If input is a DictConfig backed by dataclasses (i.e. omegaconf's structured config),
|
||||||
|
# instantiate it to the actual dataclass.
|
||||||
|
if isinstance(cfg, DictConfig) and dataclasses.is_dataclass(cfg._metadata.object_type):
|
||||||
|
return OmegaConf.to_object(cfg)
|
||||||
|
|
||||||
|
if isinstance(cfg, abc.Mapping) and "_target_" in cfg:
|
||||||
|
# conceptually equivalent to hydra.utils.instantiate(cfg) with _convert_=all,
|
||||||
|
# but faster: https://github.com/facebookresearch/hydra/issues/1200
|
||||||
|
cfg = {k: instantiate(v) for k, v in cfg.items()}
|
||||||
|
cls = cfg.pop("_target_")
|
||||||
|
cls = instantiate(cls)
|
||||||
|
|
||||||
|
if isinstance(cls, str):
|
||||||
|
cls_name = cls
|
||||||
|
cls = locate(cls_name)
|
||||||
|
assert cls is not None, cls_name
|
||||||
|
else:
|
||||||
|
try:
|
||||||
|
cls_name = cls.__module__ + "." + cls.__qualname__
|
||||||
|
except Exception:
|
||||||
|
# target could be anything, so the above could fail
|
||||||
|
cls_name = str(cls)
|
||||||
|
assert callable(cls), f"_target_ {cls} does not define a callable object"
|
||||||
|
try:
|
||||||
|
return cls(**cfg)
|
||||||
|
except TypeError:
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
logger.error(f"Error when instantiating {cls_name}!")
|
||||||
|
raise
|
||||||
|
return cfg # return as-is if don't know what to do
|
||||||
449
lazy_config/utils/config/lazy.py
Normal file
449
lazy_config/utils/config/lazy.py
Normal file
@ -0,0 +1,449 @@
|
|||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
# Original Copyright (c) Facebook, Inc. and its affiliates.
|
||||||
|
# This file has been modified by Yuyao Huang (C) 2025.
|
||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
#
|
||||||
|
# This file is based on code from Detectron2 (https://github.com/facebookresearch/detectron2)
|
||||||
|
# and is licensed under the Apache License, Version 2.0.
|
||||||
|
# You may obtain a copy of the License at (http://www.apache.org/licenses/LICENSE-2.0).
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
|
||||||
|
import ast
|
||||||
|
import builtins
|
||||||
|
import collections.abc as abc
|
||||||
|
import importlib
|
||||||
|
import inspect
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
import uuid
|
||||||
|
from contextlib import contextmanager
|
||||||
|
from copy import deepcopy
|
||||||
|
from dataclasses import is_dataclass
|
||||||
|
from typing import List, Tuple, Union
|
||||||
|
import cloudpickle
|
||||||
|
import yaml
|
||||||
|
from omegaconf import DictConfig, ListConfig, OmegaConf, SCMode
|
||||||
|
|
||||||
|
from lazy_config.utils.file_io import PathManager
|
||||||
|
from lazy_config.utils.registry import _convert_target_to_string
|
||||||
|
|
||||||
|
__all__ = ["LazyCall", "LazyConfig"]
|
||||||
|
|
||||||
|
|
||||||
|
class LazyCall:
|
||||||
|
"""
|
||||||
|
Wrap a callable so that when it's called, the call will not be executed,
|
||||||
|
but returns a dict that describes the call.
|
||||||
|
|
||||||
|
LazyCall object has to be called with only keyword arguments. Positional
|
||||||
|
arguments are not yet supported.
|
||||||
|
|
||||||
|
Examples:
|
||||||
|
::
|
||||||
|
from detectron2.config import instantiate, LazyCall
|
||||||
|
|
||||||
|
layer_cfg = LazyCall(nn.Conv2d)(in_channels=32, out_channels=32)
|
||||||
|
layer_cfg.out_channels = 64 # can edit it afterwards
|
||||||
|
layer = instantiate(layer_cfg)
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, target):
|
||||||
|
if not (callable(target) or isinstance(target, (str, abc.Mapping))):
|
||||||
|
raise TypeError(
|
||||||
|
f"target of LazyCall must be a callable or defines a callable! Got {target}"
|
||||||
|
)
|
||||||
|
self._target = target
|
||||||
|
|
||||||
|
def __call__(self, **kwargs):
|
||||||
|
if is_dataclass(self._target):
|
||||||
|
# omegaconf object cannot hold dataclass type
|
||||||
|
# https://github.com/omry/omegaconf/issues/784
|
||||||
|
target = _convert_target_to_string(self._target)
|
||||||
|
else:
|
||||||
|
target = self._target
|
||||||
|
kwargs["_target_"] = target
|
||||||
|
|
||||||
|
return DictConfig(content=kwargs, flags={"allow_objects": True})
|
||||||
|
|
||||||
|
|
||||||
|
def _visit_dict_config(cfg, func):
|
||||||
|
"""
|
||||||
|
Apply func recursively to all DictConfig in cfg.
|
||||||
|
"""
|
||||||
|
if isinstance(cfg, DictConfig):
|
||||||
|
func(cfg)
|
||||||
|
for v in cfg.values():
|
||||||
|
_visit_dict_config(v, func)
|
||||||
|
elif isinstance(cfg, ListConfig):
|
||||||
|
for v in cfg:
|
||||||
|
_visit_dict_config(v, func)
|
||||||
|
|
||||||
|
|
||||||
|
def _validate_py_syntax(filename):
|
||||||
|
# see also https://github.com/open-mmlab/mmcv/blob/master/mmcv/utils/config.py
|
||||||
|
with PathManager.open(filename, "r") as f:
|
||||||
|
content = f.read()
|
||||||
|
try:
|
||||||
|
ast.parse(content)
|
||||||
|
except SyntaxError as e:
|
||||||
|
raise SyntaxError(f"Config file {filename} has syntax error!") from e
|
||||||
|
|
||||||
|
|
||||||
|
def _cast_to_config(obj):
|
||||||
|
# if given a dict, return DictConfig instead
|
||||||
|
if isinstance(obj, dict):
|
||||||
|
return DictConfig(obj, flags={"allow_objects": True})
|
||||||
|
return obj
|
||||||
|
|
||||||
|
|
||||||
|
_CFG_PACKAGE_NAME = "detectron2._cfg_loader"
|
||||||
|
"""
|
||||||
|
A namespace to put all imported config into.
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
def _random_package_name(filename):
|
||||||
|
# generate a random package name when loading config files
|
||||||
|
return _CFG_PACKAGE_NAME + str(uuid.uuid4())[:4] + "." + os.path.basename(filename)
|
||||||
|
|
||||||
|
|
||||||
|
@contextmanager
|
||||||
|
def _patch_import():
|
||||||
|
"""
|
||||||
|
Enhance relative import statements in config files, so that they:
|
||||||
|
1. locate files purely based on relative location, regardless of packages.
|
||||||
|
e.g. you can import file without having __init__
|
||||||
|
2. do not cache modules globally; modifications of module states has no side effect
|
||||||
|
3. support other storage system through PathManager, so config files can be in the cloud
|
||||||
|
4. imported dict are turned into omegaconf.DictConfig automatically
|
||||||
|
"""
|
||||||
|
old_import = builtins.__import__
|
||||||
|
|
||||||
|
def find_relative_file(original_file, relative_import_path, level):
|
||||||
|
# NOTE: "from . import x" is not handled. Because then it's unclear
|
||||||
|
# if such import should produce `x` as a python module or DictConfig.
|
||||||
|
# This can be discussed further if needed.
|
||||||
|
relative_import_err = """
|
||||||
|
Relative import of directories is not allowed within config files.
|
||||||
|
Within a config file, relative import can only import other config files.
|
||||||
|
""".replace(
|
||||||
|
"\n", " "
|
||||||
|
)
|
||||||
|
if not len(relative_import_path):
|
||||||
|
raise ImportError(relative_import_err)
|
||||||
|
|
||||||
|
cur_file = os.path.dirname(original_file)
|
||||||
|
for _ in range(level - 1):
|
||||||
|
cur_file = os.path.dirname(cur_file)
|
||||||
|
cur_name = relative_import_path.lstrip(".")
|
||||||
|
for part in cur_name.split("."):
|
||||||
|
cur_file = os.path.join(cur_file, part)
|
||||||
|
if not cur_file.endswith(".py"):
|
||||||
|
cur_file += ".py"
|
||||||
|
if not PathManager.isfile(cur_file):
|
||||||
|
cur_file_no_suffix = cur_file[: -len(".py")]
|
||||||
|
if PathManager.isdir(cur_file_no_suffix):
|
||||||
|
raise ImportError(f"Cannot import from {cur_file_no_suffix}." + relative_import_err)
|
||||||
|
else:
|
||||||
|
raise ImportError(
|
||||||
|
f"Cannot import name {relative_import_path} from "
|
||||||
|
f"{original_file}: {cur_file} does not exist."
|
||||||
|
)
|
||||||
|
return cur_file
|
||||||
|
|
||||||
|
def new_import(name, globals=None, locals=None, fromlist=(), level=0):
|
||||||
|
if (
|
||||||
|
# Only deal with relative imports inside config files
|
||||||
|
level != 0
|
||||||
|
and globals is not None
|
||||||
|
and (globals.get("__package__", "") or "").startswith(_CFG_PACKAGE_NAME)
|
||||||
|
):
|
||||||
|
cur_file = find_relative_file(globals["__file__"], name, level)
|
||||||
|
_validate_py_syntax(cur_file)
|
||||||
|
spec = importlib.machinery.ModuleSpec(
|
||||||
|
_random_package_name(cur_file), None, origin=cur_file
|
||||||
|
)
|
||||||
|
module = importlib.util.module_from_spec(spec)
|
||||||
|
module.__file__ = cur_file
|
||||||
|
with PathManager.open(cur_file) as f:
|
||||||
|
content = f.read()
|
||||||
|
exec(compile(content, cur_file, "exec"), module.__dict__)
|
||||||
|
for name in fromlist: # turn imported dict into DictConfig automatically
|
||||||
|
val = _cast_to_config(module.__dict__[name])
|
||||||
|
module.__dict__[name] = val
|
||||||
|
return module
|
||||||
|
return old_import(name, globals, locals, fromlist=fromlist, level=level)
|
||||||
|
|
||||||
|
builtins.__import__ = new_import
|
||||||
|
yield new_import
|
||||||
|
builtins.__import__ = old_import
|
||||||
|
|
||||||
|
|
||||||
|
class LazyConfig:
|
||||||
|
"""
|
||||||
|
Provide methods to save, load, and overrides an omegaconf config object
|
||||||
|
which may contain definition of lazily-constructed objects.
|
||||||
|
"""
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def load_rel(filename: str, keys: Union[None, str, Tuple[str, ...]] = None):
|
||||||
|
"""
|
||||||
|
Similar to :meth:`load()`, but load path relative to the caller's
|
||||||
|
source file.
|
||||||
|
|
||||||
|
This has the same functionality as a relative import, except that this method
|
||||||
|
accepts filename as a string, so more characters are allowed in the filename.
|
||||||
|
"""
|
||||||
|
caller_frame = inspect.stack()[1]
|
||||||
|
caller_fname = caller_frame[0].f_code.co_filename
|
||||||
|
assert caller_fname != "<string>", "load_rel Unable to find caller"
|
||||||
|
caller_dir = os.path.dirname(caller_fname)
|
||||||
|
filename = os.path.join(caller_dir, filename)
|
||||||
|
return LazyConfig.load(filename, keys)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def load(filename: str, keys: Union[None, str, Tuple[str, ...]] = None):
|
||||||
|
"""
|
||||||
|
Load a config file.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
filename: absolute path or relative path w.r.t. the current working directory
|
||||||
|
keys: keys to load and return. If not given, return all keys
|
||||||
|
(whose values are config objects) in a dict.
|
||||||
|
"""
|
||||||
|
has_keys = keys is not None
|
||||||
|
filename = filename.replace("/./", "/") # redundant
|
||||||
|
if os.path.splitext(filename)[1] not in [".py", ".yaml", ".yml"]:
|
||||||
|
raise ValueError(f"Config file {filename} has to be a python or yaml file.")
|
||||||
|
if filename.endswith(".py"):
|
||||||
|
_validate_py_syntax(filename)
|
||||||
|
|
||||||
|
with _patch_import():
|
||||||
|
# Record the filename
|
||||||
|
module_namespace = {
|
||||||
|
"__file__": filename,
|
||||||
|
"__package__": _random_package_name(filename),
|
||||||
|
}
|
||||||
|
with PathManager.open(filename) as f:
|
||||||
|
content = f.read()
|
||||||
|
# Compile first with filename to:
|
||||||
|
# 1. make filename appears in stacktrace
|
||||||
|
# 2. make load_rel able to find its parent's (possibly remote) location
|
||||||
|
exec(compile(content, filename, "exec"), module_namespace)
|
||||||
|
|
||||||
|
ret = module_namespace
|
||||||
|
else:
|
||||||
|
with PathManager.open(filename) as f:
|
||||||
|
obj = yaml.unsafe_load(f)
|
||||||
|
ret = OmegaConf.create(obj, flags={"allow_objects": True})
|
||||||
|
|
||||||
|
if has_keys:
|
||||||
|
if isinstance(keys, str):
|
||||||
|
return _cast_to_config(ret[keys])
|
||||||
|
else:
|
||||||
|
return tuple(_cast_to_config(ret[a]) for a in keys)
|
||||||
|
else:
|
||||||
|
if filename.endswith(".py"):
|
||||||
|
# when not specified, only load those that are config objects
|
||||||
|
ret = DictConfig(
|
||||||
|
{
|
||||||
|
name: _cast_to_config(value)
|
||||||
|
for name, value in ret.items()
|
||||||
|
if isinstance(value, (DictConfig, ListConfig, dict))
|
||||||
|
and not name.startswith("_")
|
||||||
|
},
|
||||||
|
flags={"allow_objects": True},
|
||||||
|
)
|
||||||
|
return ret
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def save(cfg, filename: str):
|
||||||
|
"""
|
||||||
|
Save a config object to a yaml file.
|
||||||
|
Note that when the config dictionary contains complex objects (e.g. lambda),
|
||||||
|
it can't be saved to yaml. In that case we will print an error and
|
||||||
|
attempt to save to a pkl file instead.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
cfg: an omegaconf config object
|
||||||
|
filename: yaml file name to save the config file
|
||||||
|
"""
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
try:
|
||||||
|
cfg = deepcopy(cfg)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
else:
|
||||||
|
# if it's deep-copyable, then...
|
||||||
|
def _replace_type_by_name(x):
|
||||||
|
if "_target_" in x and callable(x._target_):
|
||||||
|
try:
|
||||||
|
x._target_ = _convert_target_to_string(x._target_)
|
||||||
|
except AttributeError:
|
||||||
|
pass
|
||||||
|
|
||||||
|
# not necessary, but makes yaml looks nicer
|
||||||
|
_visit_dict_config(cfg, _replace_type_by_name)
|
||||||
|
|
||||||
|
save_pkl = False
|
||||||
|
try:
|
||||||
|
dict = OmegaConf.to_container(
|
||||||
|
cfg,
|
||||||
|
# Do not resolve interpolation when saving, i.e. do not turn ${a} into
|
||||||
|
# actual values when saving.
|
||||||
|
resolve=False,
|
||||||
|
# Save structures (dataclasses) in a format that can be instantiated later.
|
||||||
|
# Without this option, the type information of the dataclass will be erased.
|
||||||
|
structured_config_mode=SCMode.INSTANTIATE,
|
||||||
|
)
|
||||||
|
dumped = yaml.dump(dict, default_flow_style=None, allow_unicode=True, width=9999)
|
||||||
|
with PathManager.open(filename, "w") as f:
|
||||||
|
f.write(dumped)
|
||||||
|
|
||||||
|
try:
|
||||||
|
_ = yaml.unsafe_load(dumped) # test that it is loadable
|
||||||
|
except Exception:
|
||||||
|
logger.warning(
|
||||||
|
"The config contains objects that cannot serialize to a valid yaml. "
|
||||||
|
f"{filename} is human-readable but cannot be loaded."
|
||||||
|
)
|
||||||
|
save_pkl = True
|
||||||
|
except Exception:
|
||||||
|
logger.exception("Unable to serialize the config to yaml. Error:")
|
||||||
|
save_pkl = True
|
||||||
|
|
||||||
|
if save_pkl:
|
||||||
|
new_filename = filename + ".pkl"
|
||||||
|
try:
|
||||||
|
# retry by pickle
|
||||||
|
with PathManager.open(new_filename, "wb") as f:
|
||||||
|
cloudpickle.dump(cfg, f)
|
||||||
|
logger.warning(f"Config is saved using cloudpickle at {new_filename}.")
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def apply_overrides(cfg, overrides: List[str]):
|
||||||
|
"""
|
||||||
|
In-place override contents of cfg.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
cfg: an omegaconf config object
|
||||||
|
overrides: list of strings in the format of "a=b" to override configs.
|
||||||
|
See https://hydra.cc/docs/next/advanced/override_grammar/basic/
|
||||||
|
for syntax.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
the cfg object
|
||||||
|
"""
|
||||||
|
|
||||||
|
def safe_update(cfg, key, value):
|
||||||
|
parts = key.split(".")
|
||||||
|
for idx in range(1, len(parts)):
|
||||||
|
prefix = ".".join(parts[:idx])
|
||||||
|
v = OmegaConf.select(cfg, prefix, default=None)
|
||||||
|
if v is None:
|
||||||
|
break
|
||||||
|
if not OmegaConf.is_config(v):
|
||||||
|
raise KeyError(
|
||||||
|
f"Trying to update key {key}, but {prefix} "
|
||||||
|
f"is not a config, but has type {type(v)}."
|
||||||
|
)
|
||||||
|
OmegaConf.update(cfg, key, value, merge=True)
|
||||||
|
|
||||||
|
try:
|
||||||
|
from hydra.core.override_parser.overrides_parser import OverridesParser
|
||||||
|
|
||||||
|
has_hydra = True
|
||||||
|
except ImportError:
|
||||||
|
has_hydra = False
|
||||||
|
|
||||||
|
if has_hydra:
|
||||||
|
parser = OverridesParser.create()
|
||||||
|
overrides = parser.parse_overrides(overrides)
|
||||||
|
for o in overrides:
|
||||||
|
key = o.key_or_group
|
||||||
|
value = o.value()
|
||||||
|
if o.is_delete():
|
||||||
|
# TODO support this
|
||||||
|
raise NotImplementedError("deletion is not yet a supported override")
|
||||||
|
safe_update(cfg, key, value)
|
||||||
|
else:
|
||||||
|
# Fallback. Does not support all the features and error checking like hydra.
|
||||||
|
for o in overrides:
|
||||||
|
key, value = o.split("=")
|
||||||
|
try:
|
||||||
|
value = ast.literal_eval(value)
|
||||||
|
except NameError:
|
||||||
|
pass
|
||||||
|
safe_update(cfg, key, value)
|
||||||
|
return cfg
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def to_py(cfg, prefix: str = "cfg."):
|
||||||
|
"""
|
||||||
|
Try to convert a config object into Python-like psuedo code.
|
||||||
|
|
||||||
|
Note that perfect conversion is not always possible. So the returned
|
||||||
|
results are mainly meant to be human-readable, and not meant to be executed.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
cfg: an omegaconf config object
|
||||||
|
prefix: root name for the resulting code (default: "cfg.")
|
||||||
|
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
str of formatted Python code
|
||||||
|
"""
|
||||||
|
import black
|
||||||
|
|
||||||
|
cfg = OmegaConf.to_container(cfg, resolve=True)
|
||||||
|
|
||||||
|
def _to_str(obj, prefix=None, inside_call=False):
|
||||||
|
if prefix is None:
|
||||||
|
prefix = []
|
||||||
|
if isinstance(obj, abc.Mapping) and "_target_" in obj:
|
||||||
|
# Dict representing a function call
|
||||||
|
target = _convert_target_to_string(obj.pop("_target_"))
|
||||||
|
args = []
|
||||||
|
for k, v in sorted(obj.items()):
|
||||||
|
args.append(f"{k}={_to_str(v, inside_call=True)}")
|
||||||
|
args = ", ".join(args)
|
||||||
|
call = f"{target}({args})"
|
||||||
|
return "".join(prefix) + call
|
||||||
|
elif isinstance(obj, abc.Mapping) and not inside_call:
|
||||||
|
# Dict that is not inside a call is a list of top-level config objects that we
|
||||||
|
# render as one object per line with dot separated prefixes
|
||||||
|
key_list = []
|
||||||
|
for k, v in sorted(obj.items()):
|
||||||
|
if isinstance(v, abc.Mapping) and "_target_" not in v:
|
||||||
|
key_list.append(_to_str(v, prefix=prefix + [k + "."]))
|
||||||
|
else:
|
||||||
|
key = "".join(prefix) + k
|
||||||
|
key_list.append(f"{key}={_to_str(v)}")
|
||||||
|
return "\n".join(key_list)
|
||||||
|
elif isinstance(obj, abc.Mapping):
|
||||||
|
# Dict that is inside a call is rendered as a regular dict
|
||||||
|
return (
|
||||||
|
"{"
|
||||||
|
+ ",".join(
|
||||||
|
f"{repr(k)}: {_to_str(v, inside_call=inside_call)}"
|
||||||
|
for k, v in sorted(obj.items())
|
||||||
|
)
|
||||||
|
+ "}"
|
||||||
|
)
|
||||||
|
elif isinstance(obj, list):
|
||||||
|
return "[" + ",".join(_to_str(x, inside_call=inside_call) for x in obj) + "]"
|
||||||
|
else:
|
||||||
|
return repr(obj)
|
||||||
|
|
||||||
|
py_str = _to_str(cfg, prefix=[prefix])
|
||||||
|
try:
|
||||||
|
return black.format_str(py_str, mode=black.Mode())
|
||||||
|
except black.InvalidInput:
|
||||||
|
return py_str
|
||||||
54
lazy_config/utils/file_io.py
Normal file
54
lazy_config/utils/file_io.py
Normal file
@ -0,0 +1,54 @@
|
|||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
# Original Copyright (c) Facebook, Inc. and its affiliates.
|
||||||
|
# This file has been modified by Yuyao Huang (C) 2025.
|
||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
#
|
||||||
|
# This file is based on code from Detectron2 (https://github.com/facebookresearch/detectron2)
|
||||||
|
# and is licensed under the Apache License, Version 2.0.
|
||||||
|
# You may obtain a copy of the License at (http://www.apache.org/licenses/LICENSE-2.0).
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
|
||||||
|
|
||||||
|
from iopath.common.file_io import HTTPURLHandler, OneDrivePathHandler, PathHandler
|
||||||
|
from iopath.common.file_io import PathManager as PathManagerBase
|
||||||
|
|
||||||
|
__all__ = ["PathManager", "PathHandler"]
|
||||||
|
|
||||||
|
|
||||||
|
PathManager = PathManagerBase()
|
||||||
|
"""
|
||||||
|
This is a detectron2 project-specific PathManager.
|
||||||
|
We try to stay away from global PathManager in fvcore as it
|
||||||
|
introduces potential conflicts among other libraries.
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
class Detectron2Handler(PathHandler):
|
||||||
|
"""
|
||||||
|
Resolve anything that's hosted under detectron2's namespace.
|
||||||
|
"""
|
||||||
|
|
||||||
|
PREFIX = "detectron2://"
|
||||||
|
S3_DETECTRON2_PREFIX = "https://dl.fbaipublicfiles.com/detectron2/"
|
||||||
|
|
||||||
|
def _get_supported_prefixes(self):
|
||||||
|
return [self.PREFIX]
|
||||||
|
|
||||||
|
def _get_local_path(self, path, **kwargs):
|
||||||
|
name = path[len(self.PREFIX) :]
|
||||||
|
return PathManager.get_local_path(self.S3_DETECTRON2_PREFIX + name, **kwargs)
|
||||||
|
|
||||||
|
def _open(self, path, mode="r", **kwargs):
|
||||||
|
return PathManager.open(
|
||||||
|
self.S3_DETECTRON2_PREFIX + path[len(self.PREFIX) :], mode, **kwargs
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
PathManager.register_handler(HTTPURLHandler())
|
||||||
|
PathManager.register_handler(OneDrivePathHandler())
|
||||||
|
PathManager.register_handler(Detectron2Handler())
|
||||||
73
lazy_config/utils/registry.py
Normal file
73
lazy_config/utils/registry.py
Normal file
@ -0,0 +1,73 @@
|
|||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
# Original Copyright (c) Facebook, Inc. and its affiliates.
|
||||||
|
# This file has been modified by Yuyao Huang (C) 2025.
|
||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
#
|
||||||
|
# This file is based on code from Detectron2 (https://github.com/facebookresearch/detectron2)
|
||||||
|
# and is licensed under the Apache License, Version 2.0.
|
||||||
|
# You may obtain a copy of the License at (http://www.apache.org/licenses/LICENSE-2.0).
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
|
||||||
|
|
||||||
|
from typing import Any
|
||||||
|
import pydoc
|
||||||
|
|
||||||
|
"""
|
||||||
|
`locate` maps a string (typically found in config files) to callable objects.
|
||||||
|
"""
|
||||||
|
|
||||||
|
__all__ = ["locate"]
|
||||||
|
|
||||||
|
|
||||||
|
def _convert_target_to_string(t: Any) -> str:
|
||||||
|
"""
|
||||||
|
Inverse of ``locate()``.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
t: any object with ``__module__`` and ``__qualname__``
|
||||||
|
"""
|
||||||
|
module, qualname = t.__module__, t.__qualname__
|
||||||
|
|
||||||
|
# Compress the path to this object, e.g. ``module.submodule._impl.class``
|
||||||
|
# may become ``module.submodule.class``, if the later also resolves to the same
|
||||||
|
# object. This simplifies the string, and also is less affected by moving the
|
||||||
|
# class implementation.
|
||||||
|
module_parts = module.split(".")
|
||||||
|
for k in range(1, len(module_parts)):
|
||||||
|
prefix = ".".join(module_parts[:k])
|
||||||
|
candidate = f"{prefix}.{qualname}"
|
||||||
|
try:
|
||||||
|
if locate(candidate) is t:
|
||||||
|
return candidate
|
||||||
|
except ImportError:
|
||||||
|
pass
|
||||||
|
return f"{module}.{qualname}"
|
||||||
|
|
||||||
|
|
||||||
|
def locate(name: str) -> Any:
|
||||||
|
"""
|
||||||
|
Locate and return an object ``x`` using an input string ``{x.__module__}.{x.__qualname__}``,
|
||||||
|
such as "module.submodule.class_name".
|
||||||
|
|
||||||
|
Raise Exception if it cannot be found.
|
||||||
|
"""
|
||||||
|
obj = pydoc.locate(name)
|
||||||
|
|
||||||
|
# Some cases (e.g. torch.optim.sgd.SGD) not handled correctly
|
||||||
|
# by pydoc.locate. Try a private function from hydra.
|
||||||
|
if obj is None:
|
||||||
|
try:
|
||||||
|
# from hydra.utils import get_method - will print many errors
|
||||||
|
from hydra.utils import _locate
|
||||||
|
except ImportError as e:
|
||||||
|
raise ImportError(
|
||||||
|
f"Cannot dynamically locate object {name}!") from e
|
||||||
|
else:
|
||||||
|
obj = _locate(name) # it raises if fails
|
||||||
|
|
||||||
|
return obj
|
||||||
0
projects/.gitkeep
Normal file
0
projects/.gitkeep
Normal file
9
projects/lazy_config_demo/config.py
Normal file
9
projects/lazy_config_demo/config.py
Normal file
@ -0,0 +1,9 @@
|
|||||||
|
from lazy_config.utils.config import LazyCall as L
|
||||||
|
from projects.lazy_config_demo.model import SimpleModule
|
||||||
|
|
||||||
|
|
||||||
|
MODEL = L(SimpleModule)(
|
||||||
|
in_features=128,
|
||||||
|
out_features=256,
|
||||||
|
activation="sigmoid"
|
||||||
|
)
|
||||||
11
projects/lazy_config_demo/main.py
Normal file
11
projects/lazy_config_demo/main.py
Normal file
@ -0,0 +1,11 @@
|
|||||||
|
from lazy_config.utils.config import parse_args_and_configs, instantiate
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
cfg = parse_args_and_configs()
|
||||||
|
model = instantiate(cfg.MODEL)
|
||||||
|
print(model)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
14
projects/lazy_config_demo/model.py
Normal file
14
projects/lazy_config_demo/model.py
Normal file
@ -0,0 +1,14 @@
|
|||||||
|
class SimpleModule:
|
||||||
|
|
||||||
|
def __init__(self,
|
||||||
|
in_features: int,
|
||||||
|
out_features: int,
|
||||||
|
activation: str = "relu"):
|
||||||
|
self.in_features = in_features
|
||||||
|
self.out_features = out_features
|
||||||
|
self.activation = activation
|
||||||
|
print(
|
||||||
|
f"SimpleModule created: {in_features} -> {out_features}, activation: {activation}"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
38
pyproject.toml
Normal file
38
pyproject.toml
Normal file
@ -0,0 +1,38 @@
|
|||||||
|
[build-system]
|
||||||
|
requires = ["setuptools>=65.5.0", "wheel"]
|
||||||
|
build-backend = "setuptools.build_meta"
|
||||||
|
|
||||||
|
[project]
|
||||||
|
name = "lazy_config" # pip install
|
||||||
|
version = "0.1.0"
|
||||||
|
description = "..."
|
||||||
|
readme = "README.md" # 可选, 如果你有 README.md 文件
|
||||||
|
requires-python = ">=3.10"
|
||||||
|
license = {text = "Apache-2.0"} # 或者你的许可证
|
||||||
|
authors = [
|
||||||
|
{name = "Yuyao Huang (Sam)", email = "huangyuyao@outlook.com"},
|
||||||
|
]
|
||||||
|
dependencies = [
|
||||||
|
"cloudpickle",
|
||||||
|
"omegaconf",
|
||||||
|
"iopath",
|
||||||
|
"hydra-core",
|
||||||
|
]
|
||||||
|
|
||||||
|
[tool.setuptools]
|
||||||
|
py-modules = ["lazy_config"]
|
||||||
|
|
||||||
|
# 包含非 Python 文件
|
||||||
|
# [tool.setuptools.package-data]
|
||||||
|
# "package.path" = ["*.json"]
|
||||||
|
|
||||||
|
# 命令行程序入口
|
||||||
|
# [project.scripts]
|
||||||
|
# script-name = "module.path:function"
|
||||||
|
|
||||||
|
[project.optional-dependencies]
|
||||||
|
dev = [
|
||||||
|
"pytest",
|
||||||
|
"pytest-cov",
|
||||||
|
"pytest-mock",
|
||||||
|
]
|
||||||
Loading…
x
Reference in New Issue
Block a user